2024-09-17T01:12:01.185 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T01:12:01.206 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T01:12:01.287 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7908031' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f workunit: branch: squid sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 599 sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f sleep_before_teardown: 0 subset: 33/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 targets: smithi081.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNzA7XnbNgS1rVIyxuFdOMBeR7W3/Onpr3nSH95cDbx+nafoxEjCXluZt1v8Xe/9A8K2FirqhlcVaaK0ClUUUs8= smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXpscx8Fw7/J0CBQy1KDNGDpiq7FcSL5iIOwNCfpnFyJzpW91U4c1/TKQXDUrNHZ3LvOQQ4g1Ukfr6kCz+9pAU= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_21:08:04 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-17T01:12:01.287 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa; will attempt to use it 2024-09-17T01:12:01.288 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa/tasks 2024-09-17T01:12:01.288 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T01:12:01.289 INFO:teuthology.task.internal:Checking packages... 2024-09-17T01:12:01.310 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c2ddf9721ef254645201cdbabd9c8f49c30a586f' 2024-09-17T01:12:01.311 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T01:12:01.311 INFO:teuthology.packaging:ref: None 2024-09-17T01:12:01.311 INFO:teuthology.packaging:tag: None 2024-09-17T01:12:01.311 INFO:teuthology.packaging:branch: squid 2024-09-17T01:12:01.311 INFO:teuthology.packaging:sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:12:01.311 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-17T01:12:01.592 INFO:teuthology.task.internal:Found packages for ceph version 19.2.0-0 2024-09-17T01:12:01.594 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T01:12:01.602 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T01:12:01.602 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T01:12:01.648 INFO:teuthology.task.internal:Saving configuration 2024-09-17T01:12:01.667 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T01:12:01.678 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T01:12:01.702 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi081.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 01:07:12.394312', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNzA7XnbNgS1rVIyxuFdOMBeR7W3/Onpr3nSH95cDbx+nafoxEjCXluZt1v8Xe/9A8K2FirqhlcVaaK0ClUUUs8='} 2024-09-17T01:12:01.724 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 01:07:12.396302', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXpscx8Fw7/J0CBQy1KDNGDpiq7FcSL5iIOwNCfpnFyJzpW91U4c1/TKQXDUrNHZ3LvOQQ4g1Ukfr6kCz+9pAU='} 2024-09-17T01:12:01.725 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T01:12:01.736 INFO:teuthology.task.internal:roles: ubuntu@smithi081.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-17T01:12:01.736 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-17T01:12:01.736 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T01:12:01.830 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f90d97e3ac0>, signals=[15]) 2024-09-17T01:12:01.830 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T01:12:01.840 INFO:teuthology.task.internal:Opening connections... 2024-09-17T01:12:01.840 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T01:12:01.842 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi081.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:12:01.918 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-09-17T01:12:01.919 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:12:01.995 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T01:12:02.003 DEBUG:teuthology.orchestra.run.smithi081:> uname -m 2024-09-17T01:12:02.022 INFO:teuthology.orchestra.run.smithi081.stdout:x86_64 2024-09-17T01:12:02.022 DEBUG:teuthology.orchestra.run.smithi081:> cat /etc/os-release 2024-09-17T01:12:02.080 INFO:teuthology.orchestra.run.smithi081.stdout:NAME="CentOS Stream" 2024-09-17T01:12:02.080 INFO:teuthology.orchestra.run.smithi081.stdout:VERSION="9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:ID="centos" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:ID_LIKE="rhel fedora" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:VERSION_ID="9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:PLATFORM_ID="platform:el9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:ANSI_COLOR="0;31" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:LOGO="fedora-logo-icon" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:HOME_URL="https://centos.org/" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T01:12:02.081 INFO:teuthology.orchestra.run.smithi081.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T01:12:02.082 INFO:teuthology.lock.ops:Updating smithi081.front.sepia.ceph.com on lock server 2024-09-17T01:12:02.108 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-09-17T01:12:02.126 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-09-17T01:12:02.127 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-09-17T01:12:02.185 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="CentOS Stream" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="9" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:ID="centos" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="rhel fedora" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="9" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el9" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-09-17T01:12:02.186 INFO:teuthology.orchestra.run.smithi105.stdout:LOGO="fedora-logo-icon" 2024-09-17T01:12:02.187 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T01:12:02.187 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://centos.org/" 2024-09-17T01:12:02.187 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T01:12:02.187 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T01:12:02.187 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T01:12:02.187 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-09-17T01:12:02.209 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T01:12:02.219 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T01:12:02.262 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T01:12:02.262 DEBUG:teuthology.orchestra.run.smithi081:> test '!' -e /home/ubuntu/cephtest 2024-09-17T01:12:02.265 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-09-17T01:12:02.282 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T01:12:02.306 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T01:12:02.306 DEBUG:teuthology.orchestra.run.smithi081:> test -z $(ls -A /var/lib/ceph) 2024-09-17T01:12:02.323 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-09-17T01:12:02.347 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T01:12:02.408 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T01:12:02.441 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T01:12:02.441 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T01:12:02.441 DEBUG:teuthology.orchestra.run.smithi081:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T01:12:02.442 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T01:12:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:12:02.458 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-17T01:12:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:12:02.459 DEBUG:teuthology.orchestra.run.smithi081:> uname -r 2024-09-17T01:12:02.516 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-503.el9.x86_64 2024-09-17T01:12:02.516 INFO:teuthology.task.kernel:Running kernel on smithi105: 5.14.0-503.el9.x86_64 2024-09-17T01:12:02.516 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum install -y kernel 2024-09-17T01:12:02.517 INFO:teuthology.orchestra.run.smithi081.stdout:5.14.0-503.el9.x86_64 2024-09-17T01:12:02.517 INFO:teuthology.task.kernel:Running kernel on smithi081: 5.14.0-503.el9.x86_64 2024-09-17T01:12:02.518 DEBUG:teuthology.orchestra.run.smithi081:> sudo yum install -y kernel 2024-09-17T01:12:04.948 INFO:teuthology.orchestra.run.smithi081.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.3 MB 00:00 2024-09-17T01:12:06.993 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.3 MB 00:02 2024-09-17T01:12:08.696 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-09-17T01:12:09.046 INFO:teuthology.orchestra.run.smithi081.stdout:CentOS Stream 9 - AppStream 6.7 MB/s | 20 MB 00:03 2024-09-17T01:12:14.737 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 2.9 MB/s | 6.5 MB 00:02 2024-09-17T01:12:15.785 INFO:teuthology.orchestra.run.smithi081.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 6.5 MB 00:02 2024-09-17T01:12:16.362 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 46 kB/s | 19 kB 00:00 2024-09-17T01:12:17.485 INFO:teuthology.orchestra.run.smithi081.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 19 kB 00:00 2024-09-17T01:12:18.992 INFO:teuthology.orchestra.run.smithi081.stdout:Extra Packages for Enterprise Linux 16 MB/s | 23 MB 00:01 2024-09-17T01:12:19.052 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 8.8 MB/s | 23 MB 00:02 2024-09-17T01:12:24.454 INFO:teuthology.orchestra.run.smithi081.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-17T01:12:24.765 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 19 kB/s | 1.7 kB 00:00 2024-09-17T01:12:26.335 INFO:teuthology.orchestra.run.smithi081.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T01:12:26.335 INFO:teuthology.orchestra.run.smithi081.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T01:12:26.440 INFO:teuthology.orchestra.run.smithi081.stdout:Dependencies resolved. 2024-09-17T01:12:26.452 INFO:teuthology.orchestra.run.smithi081.stdout:================================================================================ 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout: Package Architecture Version Repository Size 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout:================================================================================ 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout:Installing: 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout:Installing dependencies: 2024-09-17T01:12:26.453 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout:Transaction Summary 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout:================================================================================ 2024-09-17T01:12:26.454 INFO:teuthology.orchestra.run.smithi081.stdout:Install 4 Packages 2024-09-17T01:12:26.455 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:12:26.455 INFO:teuthology.orchestra.run.smithi081.stdout:Total download size: 79 M 2024-09-17T01:12:26.455 INFO:teuthology.orchestra.run.smithi081.stdout:Installed size: 126 M 2024-09-17T01:12:26.455 INFO:teuthology.orchestra.run.smithi081.stdout:Downloading Packages: 2024-09-17T01:12:26.755 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T01:12:26.755 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T01:12:26.857 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T01:12:26.864 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-17T01:12:26.865 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:12:26.866 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 79 M 2024-09-17T01:12:26.866 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 126 M 2024-09-17T01:12:26.866 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-17T01:12:27.188 INFO:teuthology.orchestra.run.smithi081.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 60 kB/s | 36 kB 00:00 2024-09-17T01:12:27.267 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 127 kB/s | 36 kB 00:00 2024-09-17T01:12:29.938 INFO:teuthology.orchestra.run.smithi081.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 10 MB/s | 35 MB 00:03 2024-09-17T01:12:30.188 INFO:teuthology.orchestra.run.smithi081.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 4.3 MB/s | 16 MB 00:03 2024-09-17T01:12:31.689 INFO:teuthology.orchestra.run.smithi081.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 6.4 MB/s | 29 MB 00:04 2024-09-17T01:12:31.689 INFO:teuthology.orchestra.run.smithi081.stdout:-------------------------------------------------------------------------------- 2024-09-17T01:12:31.689 INFO:teuthology.orchestra.run.smithi081.stdout:Total 15 MB/s | 79 MB 00:05 2024-09-17T01:12:32.373 INFO:teuthology.orchestra.run.smithi081.stdout:Running transaction check 2024-09-17T01:12:32.719 INFO:teuthology.orchestra.run.smithi081.stdout:Transaction check succeeded. 2024-09-17T01:12:32.720 INFO:teuthology.orchestra.run.smithi081.stdout:Running transaction test 2024-09-17T01:12:33.695 INFO:teuthology.orchestra.run.smithi081.stdout:Transaction test succeeded. 2024-09-17T01:12:33.696 INFO:teuthology.orchestra.run.smithi081.stdout:Running transaction 2024-09-17T01:12:36.153 INFO:teuthology.orchestra.run.smithi081.stdout: Preparing : 1/1 2024-09-17T01:12:36.811 INFO:teuthology.orchestra.run.smithi081.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T01:12:36.892 INFO:teuthology.orchestra.run.smithi081.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:12:39.670 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:12:39.812 INFO:teuthology.orchestra.run.smithi081.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:12:41.791 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 1.1 MB/s | 16 MB 00:14 2024-09-17T01:12:44.660 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:12:44.812 INFO:teuthology.orchestra.run.smithi081.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:12:48.098 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 1.6 MB/s | 35 MB 00:21 2024-09-17T01:12:48.578 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 1.3 MB/s | 29 MB 00:21 2024-09-17T01:12:48.578 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-17T01:12:48.578 INFO:teuthology.orchestra.run.smithi105.stdout:Total 3.6 MB/s | 79 MB 00:21 2024-09-17T01:12:49.259 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-17T01:12:49.669 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:12:49.687 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-17T01:12:49.687 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-17T01:12:50.622 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-17T01:12:50.622 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-17T01:12:53.064 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-17T01:12:53.704 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T01:12:53.793 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:12:56.551 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:12:56.686 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:13:01.500 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:13:01.537 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:06.430 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:28.054 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:28.094 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:28.422 INFO:teuthology.orchestra.run.smithi081.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:28.423 INFO:teuthology.orchestra.run.smithi081.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T01:13:28.423 INFO:teuthology.orchestra.run.smithi081.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:13:28.423 INFO:teuthology.orchestra.run.smithi081.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout:Installed: 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:13:29.321 INFO:teuthology.orchestra.run.smithi081.stdout:Complete! 2024-09-17T01:13:29.673 DEBUG:teuthology.orchestra.run.smithi081:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T01:13:29.751 INFO:teuthology.orchestra.run.smithi081.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.752 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T01:13:29.752 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T01:13:29.752 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T01:13:29.752 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T01:13:29.752 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-17T01:13:29.752 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi081.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T01:13:29.752 DEBUG:teuthology.orchestra.run.smithi081:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T01:13:29.811 INFO:teuthology.orchestra.run.smithi081.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:29.811 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T01:13:29.811 DEBUG:teuthology.orchestra.run.smithi081:> sudo rpm -qi grub2-tools 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Name : grub2-tools 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Epoch : 1 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Version : 2.06 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Release : 82.el9 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Architecture: x86_64 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T01:13:29.843 INFO:teuthology.orchestra.run.smithi081.stdout:Group : Unspecified 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Size : 8274670 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:License : GPLv3+ 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Packager : builder@centos.org 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Vendor : CentOS 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Summary : Support tools for GRUB. 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:Description : 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:13:29.844 INFO:teuthology.orchestra.run.smithi081.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T01:13:29.845 INFO:teuthology.orchestra.run.smithi081.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T01:13:29.845 INFO:teuthology.orchestra.run.smithi081.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T01:13:29.845 INFO:teuthology.orchestra.run.smithi081.stdout:hardware devices. 2024-09-17T01:13:29.845 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:13:29.845 INFO:teuthology.orchestra.run.smithi081.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T01:13:29.846 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T01:13:29.846 INFO:teuthology.task.kernel:Updating grub on smithi081 to boot 5.14.0-508.el9.x86_64 2024-09-17T01:13:29.846 DEBUG:teuthology.orchestra.run.smithi081:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T01:13:30.515 INFO:teuthology.orchestra.run.smithi081.stderr:Generating grub configuration file ... 2024-09-17T01:13:31.906 INFO:teuthology.orchestra.run.smithi081.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T01:13:31.934 INFO:teuthology.orchestra.run.smithi081.stderr:done 2024-09-17T01:13:31.940 DEBUG:teuthology.orchestra.run.smithi081:> mktemp 2024-09-17T01:13:31.956 INFO:teuthology.orchestra.run.smithi081.stdout:/tmp/tmp.WQGAiWTKxx 2024-09-17T01:13:31.957 DEBUG:teuthology.orchestra.run.smithi081:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.WQGAiWTKxx 2024-09-17T01:13:32.023 DEBUG:teuthology.orchestra.run.smithi081:> sudo chmod 0666 /tmp/tmp.WQGAiWTKxx 2024-09-17T01:13:32.159 DEBUG:teuthology.orchestra.remote:smithi081:/tmp/tmp.WQGAiWTKxx is 6KB 2024-09-17T01:13:32.208 DEBUG:teuthology.orchestra.run.smithi081:> rm -fr /tmp/tmp.WQGAiWTKxx 2024-09-17T01:13:32.223 DEBUG:teuthology.orchestra.run.smithi081:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T01:13:32.289 INFO:teuthology.orchestra.run.smithi081.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T01:13:32.289 INFO:teuthology.orchestra.run.smithi081.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T01:13:32.290 INFO:teuthology.orchestra.run.smithi081.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T01:13:32.290 INFO:teuthology.orchestra.run.smithi081.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T01:13:32.290 DEBUG:teuthology.orchestra.run.smithi081:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T01:13:32.449 DEBUG:teuthology.orchestra.run.smithi081:> sudo shutdown -r now 2024-09-17T01:13:44.604 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:44.644 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:44.975 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:44.975 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T01:13:44.975 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T01:13:44.975 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T01:13:45.820 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T01:13:45.821 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T01:13:45.821 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:13:45.821 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-17T01:13:46.192 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T01:13:46.283 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:46.284 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T01:13:46.284 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T01:13:46.284 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T01:13:46.284 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T01:13:46.285 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-17T01:13:46.285 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi105.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T01:13:46.285 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T01:13:46.343 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T01:13:46.344 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T01:13:46.344 DEBUG:teuthology.orchestra.run.smithi105:> sudo rpm -qi grub2-tools 2024-09-17T01:13:46.382 INFO:teuthology.orchestra.run.smithi105.stdout:Name : grub2-tools 2024-09-17T01:13:46.382 INFO:teuthology.orchestra.run.smithi105.stdout:Epoch : 1 2024-09-17T01:13:46.383 INFO:teuthology.orchestra.run.smithi105.stdout:Version : 2.06 2024-09-17T01:13:46.383 INFO:teuthology.orchestra.run.smithi105.stdout:Release : 82.el9 2024-09-17T01:13:46.383 INFO:teuthology.orchestra.run.smithi105.stdout:Architecture: x86_64 2024-09-17T01:13:46.383 INFO:teuthology.orchestra.run.smithi105.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T01:13:46.383 INFO:teuthology.orchestra.run.smithi105.stdout:Group : Unspecified 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:Size : 8274670 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:License : GPLv3+ 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T01:13:46.384 INFO:teuthology.orchestra.run.smithi105.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout:Packager : builder@centos.org 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout:Vendor : CentOS 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout:Summary : Support tools for GRUB. 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout:Description : 2024-09-17T01:13:46.385 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout:hardware devices. 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:13:46.386 INFO:teuthology.orchestra.run.smithi105.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T01:13:46.388 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T01:13:46.388 INFO:teuthology.task.kernel:Updating grub on smithi105 to boot 5.14.0-508.el9.x86_64 2024-09-17T01:13:46.388 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T01:13:46.986 INFO:teuthology.orchestra.run.smithi105.stderr:Generating grub configuration file ... 2024-09-17T01:13:48.335 INFO:teuthology.orchestra.run.smithi105.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T01:13:48.367 INFO:teuthology.orchestra.run.smithi105.stderr:done 2024-09-17T01:13:48.369 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-17T01:13:48.385 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.l4CuAscBvw 2024-09-17T01:13:48.385 DEBUG:teuthology.orchestra.run.smithi105:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.l4CuAscBvw 2024-09-17T01:13:48.452 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.l4CuAscBvw 2024-09-17T01:13:48.589 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.l4CuAscBvw is 6KB 2024-09-17T01:13:48.638 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.l4CuAscBvw 2024-09-17T01:13:48.653 DEBUG:teuthology.orchestra.run.smithi105:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T01:13:48.715 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T01:13:48.716 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T01:13:48.716 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T01:13:48.716 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T01:13:48.717 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T01:13:48.855 DEBUG:teuthology.orchestra.run.smithi105:> sudo shutdown -r now 2024-09-17T01:14:02.465 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-17T01:14:02.465 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:14:02.466 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi081.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:14:18.875 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-17T01:14:18.876 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:14:18.876 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:14:20.865 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.81 2024-09-17T01:14:29.874 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:14:29.875 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi081.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:14:32.925 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.81 2024-09-17T01:14:37.245 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-17T01:14:44.933 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:14:44.934 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi081.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:14:46.248 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:14:46.248 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:14:49.309 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-17T01:15:01.321 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:15:01.321 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:15:16.744 DEBUG:teuthology.orchestra.run.smithi081:> true 2024-09-17T01:15:17.351 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:15:17.351 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T01:15:17.352 DEBUG:teuthology.orchestra.run.smithi081:> uname -r 2024-09-17T01:15:17.390 INFO:teuthology.orchestra.run.smithi081.stdout:5.14.0-508.el9.x86_64 2024-09-17T01:15:17.391 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T01:15:17.391 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T01:15:17.391 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T01:15:18.392 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-17T01:15:18.393 DEBUG:teuthology.orchestra.run.smithi081:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T01:15:18.452 INFO:teuthology.orchestra.run.smithi081.stdout:ttyS1 2024-09-17T01:15:18.478 DEBUG:teuthology.parallel:result is None 2024-09-17T01:15:33.546 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-17T01:15:34.033 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:15:34.033 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T01:15:34.033 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-17T01:15:34.073 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-508.el9.x86_64 2024-09-17T01:15:34.074 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T01:15:34.074 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T01:15:34.074 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T01:15:35.075 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-17T01:15:35.075 DEBUG:teuthology.orchestra.run.smithi105:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T01:15:35.127 INFO:teuthology.orchestra.run.smithi105.stdout:ttyS1 2024-09-17T01:15:35.248 DEBUG:teuthology.parallel:result is None 2024-09-17T01:15:35.248 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T01:15:35.255 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T01:15:35.255 DEBUG:teuthology.orchestra.run.smithi081:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T01:15:35.258 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T01:15:35.274 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T01:15:35.284 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T01:15:35.318 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T01:15:35.318 DEBUG:teuthology.orchestra.run.smithi081:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T01:15:35.321 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T01:15:35.368 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T01:15:35.378 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T01:15:35.378 DEBUG:teuthology.orchestra.run.smithi081:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T01:15:35.380 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T01:15:35.413 INFO:teuthology.orchestra.run.smithi081.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T01:15:35.419 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T01:15:35.425 INFO:teuthology.orchestra.run.smithi081.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T01:15:35.429 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T01:15:35.431 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T01:15:35.437 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T01:15:35.438 DEBUG:teuthology.orchestra.run.smithi081:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T01:15:35.469 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T01:15:35.496 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T01:15:35.505 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T01:15:35.505 DEBUG:teuthology.orchestra.run.smithi081:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T01:15:35.537 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T01:15:35.552 DEBUG:teuthology.orchestra.run.smithi081:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T01:15:35.636 DEBUG:teuthology.orchestra.run.smithi081:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T01:15:35.685 DEBUG:teuthology.orchestra.run.smithi081:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T01:15:35.768 DEBUG:teuthology.orchestra.run.smithi081:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T01:15:35.811 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:15:35.811 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T01:15:35.877 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T01:15:35.917 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T01:15:35.963 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T01:15:36.046 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T01:15:36.086 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:15:36.086 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T01:15:36.151 DEBUG:teuthology.orchestra.run.smithi081:> sudo service rsyslog restart 2024-09-17T01:15:36.154 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-09-17T01:15:36.210 INFO:teuthology.orchestra.run.smithi081.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T01:15:36.237 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T01:15:36.742 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T01:15:36.752 INFO:teuthology.task.internal:Starting timer... 2024-09-17T01:15:36.752 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T01:15:36.789 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T01:15:36.864 DEBUG:teuthology.orchestra.run.smithi081:> sudo service auditd rotate 2024-09-17T01:15:36.924 INFO:teuthology.orchestra.run.smithi081.stdout:Rotating logs: 2024-09-17T01:15:36.927 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-09-17T01:15:36.980 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-09-17T01:15:36.982 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T01:15:36.982 DEBUG:teuthology.orchestra.run.smithi081:> /usr/sbin/getenforce 2024-09-17T01:15:37.003 INFO:teuthology.orchestra.run.smithi081.stdout:Permissive 2024-09-17T01:15:37.004 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-09-17T01:15:37.041 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-09-17T01:15:37.041 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi081.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi105.front.sepia.ceph.com': 'permissive'} 2024-09-17T01:15:37.042 DEBUG:teuthology.orchestra.run.smithi081:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T01:15:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:15:37.072 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T01:15:37.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:15:37.107 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T01:15:37.107 DEBUG:teuthology.orchestra.run.smithi081:> sudo /usr/sbin/setenforce permissive 2024-09-17T01:15:37.144 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-09-17T01:15:37.182 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T01:15:37.191 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T01:15:37.203 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T01:15:37.204 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi081.front.sepia.ceph.com,smithi105.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T01:19:21.837 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi081.front.sepia.ceph.com'), Remote(name='ubuntu@smithi105.front.sepia.ceph.com')] 2024-09-17T01:19:21.838 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:19:21.840 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi081.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:19:21.916 DEBUG:teuthology.orchestra.run.smithi081:> true 2024-09-17T01:19:21.995 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi081.front.sepia.ceph.com' 2024-09-17T01:19:21.996 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:19:21.996 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T01:19:22.076 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-17T01:19:22.152 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T01:19:22.152 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T01:19:22.164 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T01:19:22.164 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T01:19:22.164 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T01:19:22.167 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T01:19:22.167 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T01:19:22.193 INFO:teuthology.orchestra.run.smithi081.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T01:19:22.207 INFO:teuthology.orchestra.run.smithi081.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T01:19:22.221 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T01:19:22.231 INFO:teuthology.orchestra.run.smithi081.stderr:sudo: ntpd: command not found 2024-09-17T01:19:22.235 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T01:19:22.243 INFO:teuthology.orchestra.run.smithi081.stdout:506 Cannot talk to daemon 2024-09-17T01:19:22.256 INFO:teuthology.orchestra.run.smithi081.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T01:19:22.260 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-09-17T01:19:22.269 INFO:teuthology.orchestra.run.smithi081.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T01:19:22.271 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-09-17T01:19:22.285 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T01:19:22.300 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T01:19:22.328 INFO:teuthology.orchestra.run.smithi081.stderr:bash: line 1: ntpq: command not found 2024-09-17T01:19:22.331 INFO:teuthology.orchestra.run.smithi081.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T01:19:22.331 INFO:teuthology.orchestra.run.smithi081.stdout:=============================================================================== 2024-09-17T01:19:22.331 INFO:teuthology.orchestra.run.smithi081.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.331 INFO:teuthology.orchestra.run.smithi081.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.332 INFO:teuthology.orchestra.run.smithi081.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.332 INFO:teuthology.orchestra.run.smithi081.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.353 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-17T01:19:22.356 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T01:19:22.356 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-17T01:19:22.357 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.357 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.357 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.357 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:19:22.358 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T01:19:22.444 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'} 2024-09-17T01:19:22.444 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-17T01:19:22.445 INFO:tasks.cephadm:Cluster fsid is ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:22.445 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T01:19:22.445 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.81', 'mon.c': '[v2:172.21.15.81:3301,v1:172.21.15.81:6790]', 'mon.b': '172.21.15.105'} 2024-09-17T01:19:22.445 INFO:tasks.cephadm:First mon is mon.a on smithi081 2024-09-17T01:19:22.445 INFO:tasks.cephadm:First mgr is y 2024-09-17T01:19:22.445 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T01:19:22.446 DEBUG:teuthology.orchestra.run.smithi081:> sudo hostname $(hostname -s) 2024-09-17T01:19:22.472 DEBUG:teuthology.orchestra.run.smithi105:> sudo hostname $(hostname -s) 2024-09-17T01:19:22.498 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-17T01:19:22.498 DEBUG:teuthology.orchestra.run.smithi081:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.717 INFO:teuthology.orchestra.run.smithi081.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 01:19 /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.718 DEBUG:teuthology.orchestra.run.smithi105:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.945 INFO:teuthology.orchestra.run.smithi105.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 01:19 /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.946 DEBUG:teuthology.orchestra.run.smithi081:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.963 DEBUG:teuthology.orchestra.run.smithi105:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T01:19:22.989 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-17T01:19:22.990 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T01:19:23.005 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T01:19:24.635 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T01:19:24.656 INFO:teuthology.orchestra.run.smithi081.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T01:19:35.950 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:19:35.950 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T01:19:35.950 INFO:teuthology.orchestra.run.smithi081.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T01:19:35.950 INFO:teuthology.orchestra.run.smithi081.stdout: "repo_digests": [ 2024-09-17T01:19:35.951 INFO:teuthology.orchestra.run.smithi081.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T01:19:35.951 INFO:teuthology.orchestra.run.smithi081.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T01:19:35.951 INFO:teuthology.orchestra.run.smithi081.stdout: ] 2024-09-17T01:19:35.951 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: "repo_digests": [ 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T01:19:36.677 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-17T01:19:36.678 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-17T01:19:36.695 DEBUG:teuthology.orchestra.run.smithi081:> sudo mkdir -p /etc/ceph 2024-09-17T01:19:36.727 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph 2024-09-17T01:19:36.754 DEBUG:teuthology.orchestra.run.smithi081:> sudo chmod 777 /etc/ceph 2024-09-17T01:19:36.796 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /etc/ceph 2024-09-17T01:19:36.820 INFO:tasks.cephadm:Writing seed config... 2024-09-17T01:19:36.821 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-17T01:19:36.821 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T01:19:36.821 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T01:19:36.822 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T01:19:36.822 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:19:36.823 DEBUG:teuthology.orchestra.run.smithi081:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T01:19:36.853 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ded6f264-7492-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T01:19:36.854 DEBUG:teuthology.orchestra.run.smithi081:mon.a> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service 2024-09-17T01:19:36.897 DEBUG:teuthology.orchestra.run.smithi081:mgr.y> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service 2024-09-17T01:19:36.940 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T01:19:36.940 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid ded6f264-7492-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.81 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:19:37.129 INFO:teuthology.orchestra.run.smithi081.stderr:-------------------------------------------------------------------------------- 2024-09-17T01:19:37.129 INFO:teuthology.orchestra.run.smithi081.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'ded6f264-7492-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.81', '--skip-admin-label'] 2024-09-17T01:19:37.150 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: 5.2.2 2024-09-17T01:19:37.152 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying podman|docker is present... 2024-09-17T01:19:37.173 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: 5.2.2 2024-09-17T01:19:37.176 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying lvm2 is present... 2024-09-17T01:19:37.176 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying time synchronization is in place... 2024-09-17T01:19:37.182 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T01:19:37.189 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:19:37.195 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: enabled 2024-09-17T01:19:37.202 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: active 2024-09-17T01:19:37.202 INFO:teuthology.orchestra.run.smithi081.stderr:Unit chronyd.service is enabled and running 2024-09-17T01:19:37.202 INFO:teuthology.orchestra.run.smithi081.stderr:Repeating the final host check... 2024-09-17T01:19:37.222 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: 5.2.2 2024-09-17T01:19:37.224 INFO:teuthology.orchestra.run.smithi081.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-17T01:19:37.224 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl is present 2024-09-17T01:19:37.224 INFO:teuthology.orchestra.run.smithi081.stderr:lvcreate is present 2024-09-17T01:19:37.230 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T01:19:37.236 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:19:37.243 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: enabled 2024-09-17T01:19:37.249 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: active 2024-09-17T01:19:37.250 INFO:teuthology.orchestra.run.smithi081.stderr:Unit chronyd.service is enabled and running 2024-09-17T01:19:37.250 INFO:teuthology.orchestra.run.smithi081.stderr:Host looks OK 2024-09-17T01:19:37.250 INFO:teuthology.orchestra.run.smithi081.stderr:Cluster fsid: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:37.250 INFO:teuthology.orchestra.run.smithi081.stderr:Acquiring lock 140589371191936 on /run/cephadm/ded6f264-7492-11ef-bceb-c7b262605968.lock 2024-09-17T01:19:37.250 INFO:teuthology.orchestra.run.smithi081.stderr:Lock 140589371191936 acquired on /run/cephadm/ded6f264-7492-11ef-bceb-c7b262605968.lock 2024-09-17T01:19:37.251 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying IP 172.21.15.81 port 3300 ... 2024-09-17T01:19:37.251 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying IP 172.21.15.81 port 6789 ... 2024-09-17T01:19:37.251 INFO:teuthology.orchestra.run.smithi081.stderr:Base mon IP is 172.21.15.81, final addrv is [v2:172.21.15.81:3300,v1:172.21.15.81:6789] 2024-09-17T01:19:37.253 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.81 metric 100 2024-09-17T01:19:37.253 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.81 metric 100 2024-09-17T01:19:37.256 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T01:19:37.256 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-17T01:19:37.256 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d753/64 scope link noprefixroute 2024-09-17T01:19:37.259 INFO:teuthology.orchestra.run.smithi081.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T01:19:37.260 INFO:teuthology.orchestra.run.smithi081.stderr:Mon IP `172.21.15.81` is in CIDR network `172.21.0.0/20` 2024-09-17T01:19:37.260 INFO:teuthology.orchestra.run.smithi081.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T01:19:37.260 INFO:teuthology.orchestra.run.smithi081.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T01:19:37.360 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Getting image source signatures 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-17T01:19:37.765 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T01:19:37.912 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T01:19:38.065 INFO:teuthology.orchestra.run.smithi081.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T01:19:38.446 INFO:teuthology.orchestra.run.smithi081.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T01:19:38.972 INFO:teuthology.orchestra.run.smithi081.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T01:19:38.972 INFO:teuthology.orchestra.run.smithi081.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T01:19:39.364 INFO:teuthology.orchestra.run.smithi081.stderr:stat: 167 167 2024-09-17T01:19:39.919 INFO:teuthology.orchestra.run.smithi081.stderr:Creating initial keys... 2024-09-17T01:19:40.319 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph-authtool: AQAs2ehmREIkDhAAQif/LGUSWqtlfZYEikUy/w== 2024-09-17T01:19:41.325 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph-authtool: AQAt2ehmpzV+DhAArMcyr0X5Vq2P+BMJhNOw7A== 2024-09-17T01:19:42.338 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph-authtool: AQAu2ehmP+hiDxAAmh4PpNbZYIB/bj3iTFHsZA== 2024-09-17T01:19:42.954 INFO:teuthology.orchestra.run.smithi081.stderr:Creating initial monmap... 2024-09-17T01:19:43.353 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T01:19:43.353 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-17T01:19:43.353 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:43.353 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr:monmaptool for a [v2:172.21.15.81:3300,v1:172.21.15.81:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr:setting min_mon_release = octopus 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: set fsid to ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:19:43.801 INFO:teuthology.orchestra.run.smithi081.stderr:Creating mon... 2024-09-17T01:19:44.825 INFO:teuthology.orchestra.run.smithi081.stderr:create mon.a on 2024-09-17T01:19:45.059 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T01:19:45.319 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target. 2024-09-17T01:19:45.320 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target. 2024-09-17T01:19:45.634 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Failed to reset failed state of unit ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-17T01:19:45.642 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target.wants/ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968@.service. 2024-09-17T01:19:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 podman[23342]: 2024-09-17 01:19:46.085535034 +0000 UTC m=+0.137045556 container create 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-17T01:19:46.373 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: disabled 2024-09-17T01:19:46.380 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:19:46.381 INFO:teuthology.orchestra.run.smithi081.stderr:firewalld.service is not enabled 2024-09-17T01:19:46.381 INFO:teuthology.orchestra.run.smithi081.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T01:19:46.382 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mon to start... 2024-09-17T01:19:46.382 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mon... 2024-09-17T01:19:46.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 podman[23342]: 2024-09-17 01:19:46.279632875 +0000 UTC m=+0.331143407 container init 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, RELEASE=HEAD, version=8, build-date=2022-05-03T08:36:31.336870) 2024-09-17T01:19:46.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 podman[23342]: 2024-09-17 01:19:46.282983836 +0000 UTC m=+0.334494367 container start 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., distribution-scope=public, version=8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream) 2024-09-17T01:19:46.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 ceph-mon[23360]: mkfs ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:46.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 ceph-mon[23360]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T01:19:46.504 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 bash[23342]: 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e 2024-09-17T01:19:46.504 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:46 smithi081 systemd[1]: Started Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: cluster: 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: id: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: services: 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.590905s) 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: data: 2024-09-17T01:19:46.910 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T01:19:46.911 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T01:19:46.911 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T01:19:46.911 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: pgs: 2024-09-17T01:19:46.911 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:47.444 INFO:teuthology.orchestra.run.smithi081.stderr:mon is available 2024-09-17T01:19:47.444 INFO:teuthology.orchestra.run.smithi081.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: monmap e1: 1 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0]} 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: fsmap 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: mgrmap e1: no daemons active 2024-09-17T01:19:47.559 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:47 smithi081 ceph-mon[23360]: from='client.? 172.21.15.81:0/3988399915' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:19:47.950 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [global] 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: fsid = ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.81:3300,v1:172.21.15.81:6789] 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [mgr] 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:19:47.951 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [osd] 2024-09-17T01:19:47.952 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T01:19:47.952 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T01:19:48.500 INFO:teuthology.orchestra.run.smithi081.stderr:Generating new minimal ceph.conf... 2024-09-17T01:19:48.616 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:48 smithi081 ceph-mon[23360]: from='client.? 172.21.15.81:0/2166458184' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T01:19:48.616 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:48 smithi081 ceph-mon[23360]: from='client.? 172.21.15.81:0/2166458184' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T01:19:49.538 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 ceph-mon[23360]: from='client.? 172.21.15.81:0/3272345191' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:19:49.540 INFO:teuthology.orchestra.run.smithi081.stderr:Restarting the monitor... 2024-09-17T01:19:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 systemd[1]: Stopping Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:19:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 bash[23578]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-17T01:19:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[23356]: 2024-09-17T01:19:49.684+0000 7f908ee1b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:19:49.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[23356]: 2024-09-17T01:19:49.684+0000 7f908ee1b700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T01:19:49.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:49 smithi081 podman[23589]: 2024-09-17 01:19:49.757221621 +0000 UTC m=+0.153586969 container died 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, ceph=True, io.buildah.version=1.19.8, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True) 2024-09-17T01:19:50.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:50 smithi081 podman[23589]: 2024-09-17 01:19:50.085275125 +0000 UTC m=+0.481640482 container cleanup 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, com.redhat.component=centos-stream-container, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, io.openshift.expose-services=, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T01:19:50.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:50 smithi081 bash[23589]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a 2024-09-17T01:19:50.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:50 smithi081 bash[23615]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon.a" found: no such container 2024-09-17T01:19:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:51 smithi081 podman[23603]: 2024-09-17 01:19:51.754861929 +0000 UTC m=+1.994118003 container remove 6db8350bd681f9acdbbcc8755c229e660b8b1966b02541064fff0ebdf71d3a3e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-09-17T01:19:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:51 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-17T01:19:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:51 smithi081 systemd[1]: Stopped Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:19:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:51 smithi081 systemd[1]: Starting Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:19:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:51 smithi081 podman[23677]: 2024-09-17 01:19:51.887956279 +0000 UTC m=+0.016424611 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T01:19:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:52 smithi081 podman[23677]: 2024-09-17 01:19:52.868953471 +0000 UTC m=+0.997421784 container create b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, distribution-scope=public, vcs-type=git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8) 2024-09-17T01:19:55.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 podman[23677]: 2024-09-17 01:19:54.885966593 +0000 UTC m=+3.014434909 container init b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, distribution-scope=public, vcs-type=git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , name=centos-stream, release=754, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-17T01:19:55.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 podman[23677]: 2024-09-17 01:19:54.890163088 +0000 UTC m=+3.018631403 container start b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, RELEASE=HEAD, version=8, ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-09-17T01:19:55.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:19:55.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T01:19:55.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: pidfile_write: ignore empty --pid-file 2024-09-17T01:19:55.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: load: jerasure load: lrc 2024-09-17T01:19:55.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: RocksDB version: 6.15.5 2024-09-17T01:19:55.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T01:19:55.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Compile date Apr 18 2022 2024-09-17T01:19:55.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: DB SUMMARY 2024-09-17T01:19:55.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: DB Session ID: AZFQG81LYNEAOZJR9ZRH 2024-09-17T01:19:55.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:19:55.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:19:55.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-17T01:19:55.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82637 ; 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:19:55.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.env: 0x55cd270c6860 2024-09-17T01:19:55.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.fs: Posix File System 2024-09-17T01:19:55.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.info_log: 0x55cd27c93dc0 2024-09-17T01:19:55.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:19:55.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.statistics: (nil) 2024-09-17T01:19:55.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.use_fsync: 0 2024-09-17T01:19:55.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:19:55.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:19:55.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:19:55.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:19:55.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:19:55.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:19:55.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:19:55.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:19:55.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.db_log_dir: 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:19:55.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:19:55.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:19:55.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:19:55.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:19:55.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:19:55.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:19:55.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:19:55.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.write_buffer_manager: 0x55cd27d842a0 2024-09-17T01:19:55.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:19:55.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:19:55.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:19:55.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:19:55.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.unordered_write: 0 2024-09-17T01:19:55.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:19:55.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:19:55.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:19:55.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:19:55.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.row_cache: None 2024-09-17T01:19:55.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.wal_filter: None 2024-09-17T01:19:55.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:19:55.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:19:55.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.preserve_deletes: 0 2024-09-17T01:19:55.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:19:55.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:19:55.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:19:55.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:19:55.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:19:55.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:19:55.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:19:55.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:19:55.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:19:55.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:19:55.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_open_files: -1 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Compression algorithms supported: 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kZSTD supported: 0 2024-09-17T01:19:55.201 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.merge_operator: 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_filter: None 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:19:55.202 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cd27c5fd00) 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_type: 0 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: data_block_index_type: 0 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_shortening: 1 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: hash_index_allow_collision: 1 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: checksum: 1 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: no_block_cache: 0 2024-09-17T01:19:55.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache: 0x55cd27cca170 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_options: 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: capacity : 536870912 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: num_shard_bits : 4 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: strict_capacity_limit : 0 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_compressed: (nil) 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: persistent_cache: (nil) 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_size: 4096 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_size_deviation: 10 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_restart_interval: 16 2024-09-17T01:19:55.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_block_restart_interval: 1 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: metadata_block_size: 4096 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: partition_filters: 0 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: use_delta_encoding: 1 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: whole_key_filtering: 1 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: verify_compression: 0 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: format_version: 4 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: enable_index_compression: 1 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_align: 0 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression: NoCompression 2024-09-17T01:19:55.205 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.num_levels: 7 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:19:55.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:19:55.207 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:19:55.208 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.table_properties_collectors: 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:19:55.209 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.ttl: 2592000 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:19:55.210 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535994919789, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535994920540, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79187, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77450, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71759, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726535994, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f70ea838-950b-444a-859d-c061f83b90cc", "db_session_id": "AZFQG81LYNEAOZJR9ZRH"}} 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535994920931, "job": 1, "event": "recovery_finished"} 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55cd27cb0a80 2024-09-17T01:19:55.211 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: DB pointer 0x55cd27cc0000 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** DB Stats ** 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:19:55.212 INFO:journalctl@ceph.mon.a.smithi081.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:19:55.213 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative compaction: 0.00 GB write, 22.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval compaction: 0.00 GB write, 22.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.214 INFO:journalctl@ceph.mon.a.smithi081.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative compaction: 0.00 GB write, 21.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:19:55.215 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: starting mon.a rank 0 at public addrs [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] at bind addrs [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???) e1 preinit fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).mds e1 new map 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).mds e1 print_map 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: e1 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: legacy client fscid: -1 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout: No filesystems configured 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:19:55.216 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expand map: {default=false} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta from 'false' to 'false' 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expanded map: {default=false} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expand map: {default=info} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta from 'info' to 'info' 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expanded map: {default=info} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expand map: {default=daemon} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expanded map: {default=daemon} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expand map: {default=debug} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta from 'debug' to 'debug' 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: expand_channel_meta expanded map: {default=debug} 2024-09-17T01:19:55.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T01:19:55.218 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: monmap e1: 1 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0]} 2024-09-17T01:19:55.218 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: fsmap 2024-09-17T01:19:55.218 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T01:19:55.218 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:54 smithi081 ceph-mon[23696]: mgrmap e1: no daemons active 2024-09-17T01:19:55.807 INFO:teuthology.orchestra.run.smithi081.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T01:19:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:55 smithi081 bash[23677]: b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 2024-09-17T01:19:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:55 smithi081 systemd[1]: Started Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:20:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:19:59 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/749507343' entity='client.admin' 2024-09-17T01:20:04.134 INFO:teuthology.orchestra.run.smithi081.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T01:20:04.135 INFO:teuthology.orchestra.run.smithi081.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:20:04.135 INFO:teuthology.orchestra.run.smithi081.stderr:Creating mgr... 2024-09-17T01:20:04.135 INFO:teuthology.orchestra.run.smithi081.stderr:Verifying port 9283 ... 2024-09-17T01:20:04.308 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Failed to reset failed state of unit ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-17T01:20:04.315 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968.target.wants/ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-ded6f264-7492-11ef-bceb-c7b262605968@.service. 2024-09-17T01:20:04.995 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: disabled 2024-09-17T01:20:05.003 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:20:05.004 INFO:teuthology.orchestra.run.smithi081.stderr:firewalld.service is not enabled 2024-09-17T01:20:05.004 INFO:teuthology.orchestra.run.smithi081.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T01:20:05.014 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: disabled 2024-09-17T01:20:05.022 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:20:05.023 INFO:teuthology.orchestra.run.smithi081.stderr:firewalld.service is not enabled 2024-09-17T01:20:05.023 INFO:teuthology.orchestra.run.smithi081.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T01:20:05.023 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mgr to start... 2024-09-17T01:20:05.024 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mgr... 2024-09-17T01:20:05.100 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:05.101+0000 7fd353dc0000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:05.366 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:05.280+0000 7fd353dc0000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:20:05.366 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:05.367+0000 7fd353dc0000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsid": "ded6f264-7492-11ef-bceb-c7b262605968", 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "health": { 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T01:20:05.602 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 0 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "a" 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.603 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T01:20:05.604 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:05.604 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T01:20:05.604 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T01:20:05.604 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T01:20:05.604 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T01:20:05.607 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": false, 2024-09-17T01:20:05.608 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "iostat", 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "nfs", 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "restful" 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modified": "2024-09-17T01:19:46.321773+0000", 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T01:20:05.609 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:05.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:05.513+0000 7fd353dc0000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:06.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3748624742' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T01:20:06.060 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:05.728+0000 7fd353dc0000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:20:06.168 INFO:teuthology.orchestra.run.smithi081.stderr:mgr not available, waiting (1/15)... 2024-09-17T01:20:06.313 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:06.061+0000 7fd353dc0000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:20:06.313 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:06.150+0000 7fd353dc0000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:20:06.313 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:06.315+0000 7fd353dc0000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:20:07.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:07.359+0000 7fd353dc0000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:20:07.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:07.446+0000 7fd353dc0000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:20:08.551 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:08.345+0000 7fd353dc0000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsid": "ded6f264-7492-11ef-bceb-c7b262605968", 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "health": { 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T01:20:08.654 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 0 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "a" 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T01:20:08.655 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T01:20:08.656 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T01:20:08.659 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": false, 2024-09-17T01:20:08.660 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "iostat", 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "nfs", 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "restful" 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modified": "2024-09-17T01:19:46.321773+0000", 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T01:20:08.661 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:08.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2768834372' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T01:20:08.881 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:08.657+0000 7fd353dc0000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:20:08.881 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:08.793+0000 7fd353dc0000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:08.881 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:08.882+0000 7fd353dc0000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:20:09.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:08.982+0000 7fd353dc0000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:20:09.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.071+0000 7fd353dc0000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:20:09.282 INFO:teuthology.orchestra.run.smithi081.stderr:mgr not available, waiting (2/15)... 2024-09-17T01:20:09.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.221+0000 7fd353dc0000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:20:09.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.314+0000 7fd353dc0000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:20:09.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.408+0000 7fd353dc0000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:20:10.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.686+0000 7fd353dc0000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:10.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.772+0000 7fd353dc0000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:20:10.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:09.860+0000 7fd353dc0000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:20:11.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:10.904+0000 7fd353dc0000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:20:11.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: Activating manager daemon y 2024-09-17T01:20:11.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: mgrmap e2: y(active, starting, since 0.00282451s) 2024-09-17T01:20:11.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:20:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:10 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsid": "ded6f264-7492-11ef-bceb-c7b262605968", 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "health": { 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T01:20:11.768 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 0 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "a" 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_age": 16, 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T01:20:11.769 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:11.770 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T01:20:11.770 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T01:20:11.770 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T01:20:11.773 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": false, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T01:20:11.774 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "iostat", 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "nfs", 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "restful" 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modified": "2024-09-17T01:19:46.321773+0000", 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T01:20:11.775 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:11 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' 2024-09-17T01:20:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:11 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' 2024-09-17T01:20:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:11 smithi081 ceph-mon[23696]: from='mgr.14100 172.21.15.81:0/3980781160' entity='mgr.y' 2024-09-17T01:20:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4199423329' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T01:20:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:11 smithi081 ceph-mon[23696]: mgrmap e3: y(active, since 1.00587s) 2024-09-17T01:20:12.279 INFO:teuthology.orchestra.run.smithi081.stderr:mgr not available, waiting (3/15)... 2024-09-17T01:20:14.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:13 smithi081 ceph-mon[23696]: mgrmap e4: y(active, since 2s) 2024-09-17T01:20:14.999 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:14.999 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:14.999 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsid": "ded6f264-7492-11ef-bceb-c7b262605968", 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "health": { 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 0 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:15.000 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "a" 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "quorum_age": 20, 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T01:20:15.001 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T01:20:15.002 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:15.003 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": true, 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "iostat", 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "nfs", 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "restful" 2024-09-17T01:20:15.004 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ], 2024-09-17T01:20:15.005 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:15.005 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.005 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T01:20:15.005 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T01:20:15.005 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "modified": "2024-09-17T01:19:46.321773+0000", 2024-09-17T01:20:15.006 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "services": {} 2024-09-17T01:20:15.006 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: }, 2024-09-17T01:20:15.006 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T01:20:15.006 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:15.553 INFO:teuthology.orchestra.run.smithi081.stderr:mgr is available 2024-09-17T01:20:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:15 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2950489883' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [global] 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: fsid = ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [mgr] 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: [osd] 2024-09-17T01:20:16.198 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T01:20:16.199 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T01:20:16.734 INFO:teuthology.orchestra.run.smithi081.stderr:Enabling cephadm module... 2024-09-17T01:20:16.949 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:16 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/313323932' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T01:20:16.949 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:16 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/313323932' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T01:20:18.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1830922409' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T01:20:18.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:17 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setuser ceph since I am not root 2024-09-17T01:20:18.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:17 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setgroup ceph since I am not root 2024-09-17T01:20:18.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.095+0000 7f4d0b6d1000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:20:18.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.173+0000 7f4d0b6d1000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:18.569 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.348+0000 7f4d0b6d1000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:20:18.569 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.435+0000 7f4d0b6d1000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:20:18.569 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.570+0000 7f4d0b6d1000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:18.824 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:18.785+0000 7f4d0b6d1000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:20:19.122 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1830922409' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T01:20:19.122 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:18 smithi081 ceph-mon[23696]: mgrmap e5: y(active, since 7s) 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": true, 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T01:20:19.158 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:19.378 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:19.123+0000 7f4d0b6d1000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:20:19.379 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:19.215+0000 7f4d0b6d1000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:20:19.379 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:19.380+0000 7f4d0b6d1000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:20:19.713 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for the mgr to restart... 2024-09-17T01:20:19.713 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mgr epoch 5... 2024-09-17T01:20:20.023 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:19 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4161907819' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T01:20:20.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:20.448+0000 7f4d0b6d1000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:20:20.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:20.538+0000 7f4d0b6d1000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:20:21.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:21 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:21.425+0000 7f4d0b6d1000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:20:22.046 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:21 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:21.732+0000 7f4d0b6d1000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:20:22.046 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:21 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:21.863+0000 7f4d0b6d1000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:22.047 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:21 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:21.951+0000 7f4d0b6d1000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:20:22.047 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.048+0000 7f4d0b6d1000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:20:22.372 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.135+0000 7f4d0b6d1000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:20:22.372 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.282+0000 7f4d0b6d1000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:20:22.372 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.374+0000 7f4d0b6d1000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:20:22.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.469+0000 7f4d0b6d1000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:20:23.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.750+0000 7f4d0b6d1000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:23.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.837+0000 7f4d0b6d1000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:20:23.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:22.924+0000 7f4d0b6d1000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:20:24.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:24.005+0000 7f4d0b6d1000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:20:24.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-mon[23696]: Active manager daemon y restarted 2024-09-17T01:20:24.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-mon[23696]: Activating manager daemon y 2024-09-17T01:20:24.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-mon[23696]: osdmap e2: 0 total, 0 up, 0 in 2024-09-17T01:20:25.059 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:24] ENGINE Bus STARTING 2024-09-17T01:20:25.059 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:24] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:20:25.059 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:24] ENGINE Bus STARTED 2024-09-17T01:20:25.071 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:25.072 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T01:20:25.072 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T01:20:25.072 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:25.367 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: mgrmap e6: y(active, starting, since 0.0525227s) 2024-09-17T01:20:25.367 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:20:25.367 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:20:25.368 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:24] ENGINE Bus STARTING 2024-09-17T01:20:25.369 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:24] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:20:25.369 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:24] ENGINE Bus STARTED 2024-09-17T01:20:25.369 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:25.369 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:25 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:25.628 INFO:teuthology.orchestra.run.smithi081.stderr:mgr epoch 5 is available 2024-09-17T01:20:25.629 INFO:teuthology.orchestra.run.smithi081.stderr:Setting orchestrator backend to cephadm... 2024-09-17T01:20:26.276 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:26 smithi081 ceph-mon[23696]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T01:20:26.277 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:26 smithi081 ceph-mon[23696]: mgrmap e7: y(active, since 1.05964s) 2024-09-17T01:20:26.277 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:26 smithi081 ceph-mon[23696]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T01:20:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:27 smithi081 ceph-mon[23696]: mgrmap e8: y(active, since 2s) 2024-09-17T01:20:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:27 smithi081 ceph-mon[23696]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:27 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:27 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:27.435 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: value unchanged 2024-09-17T01:20:28.007 INFO:teuthology.orchestra.run.smithi081.stderr:Generating ssh key... 2024-09-17T01:20:28.258 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-mon[23696]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:29.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: Generating public/private rsa key pair. 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: Your identification has been saved in /tmp/tmpxf8um23x/key. 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: Your public key has been saved in /tmp/tmpxf8um23x/key.pub. 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: The key fingerprint is: 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: SHA256:YqxSP+NWVphdsrpK/HSOxF8k+uNZMk6aqAKQkemOCW8 ceph-ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: The key's randomart image is: 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: +---[RSA 3072]----+ 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: | o | 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: |+ . . | 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: |.o + + | 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: |+. . o + | 2024-09-17T01:20:29.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: |=o . + So. . | 2024-09-17T01:20:29.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: |+.E. = o+. o | 2024-09-17T01:20:29.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: | o. . *o=.= o | 2024-09-17T01:20:29.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: | .. o.B.@.* | 2024-09-17T01:20:29.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: | ..o+.=.B. | 2024-09-17T01:20:29.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: +----[SHA256]-----+ 2024-09-17T01:20:30.037 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFYqJZL6DIkghT/W76axRvXIxjV5jSaKa6bcHlj+UB6dtxNFNLRE3/2BGSoWDDzF817MDPZ3g1OnezOTwLoflnOOH/H73GoRSXgDgA0taN4oDfgDQIPJMxxe6/tt8VnyKo+TWbOXW3KYr69Kb+CV/BWfR1BNynJHyPMpYBh3YHQTUgRNc5KhBtCU/EKJrZG3iWIaPkBUJx8GroLWlQKakwLcmvwrHVC0EcyWUKl13bCa2GMiPuRf2zCCfm4MYtJL7HZpVucTs/z6r3Ehvi2EChf3Wb5Jczu53694iqQaOS9Ae3hSrxF7EDBKtz4ah9aN0mmfm1LdsHEL8TGPWwdVfQyJNHkqH8jSFlPdUIFLYiNwCTkuSKEk3TRdbdng+VP+DnjNQvFZq8MjnNxuBxwHhiLkR0WFEFQesGBagWXxhZ1zf9D3+BrmI6/4Yvos8jogspE5mjZ9KHbV39P1nztumWyoYS4TtqJdY7WTDz00WPF23+5rNpHrCTI7VMXMqbYDE= ceph-ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:20:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:29 smithi081 ceph-mon[23696]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:29 smithi081 ceph-mon[23696]: Generating ssh key... 2024-09-17T01:20:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:29 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:29 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:30.586 INFO:teuthology.orchestra.run.smithi081.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T01:20:30.586 INFO:teuthology.orchestra.run.smithi081.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T01:20:30.593 INFO:teuthology.orchestra.run.smithi081.stderr:Adding host smithi081... 2024-09-17T01:20:31.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:31 smithi081 ceph-mon[23696]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:32.095 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: Added host 'smithi081' with addr '172.21.15.81' 2024-09-17T01:20:32.124 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:32 smithi081 ceph-mon[23696]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi081", "addr": "172.21.15.81", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:32.124 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:32 smithi081 ceph-mon[23696]: Deploying cephadm binary to smithi081 2024-09-17T01:20:32.727 INFO:teuthology.orchestra.run.smithi081.stderr:Deploying unmanaged mon service... 2024-09-17T01:20:33.355 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:33 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:33.355 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:33 smithi081 ceph-mon[23696]: Added host smithi081 2024-09-17T01:20:33.355 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:33 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:33.496 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T01:20:34.088 INFO:teuthology.orchestra.run.smithi081.stderr:Deploying unmanaged mgr service... 2024-09-17T01:20:34.700 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T01:20:34.733 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:34 smithi081 ceph-mon[23696]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:34.733 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:34 smithi081 ceph-mon[23696]: Saving service mon spec with placement count:5 2024-09-17T01:20:34.733 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:34 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:35.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:35 smithi081 ceph-mon[23696]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:35.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:35 smithi081 ceph-mon[23696]: Saving service mgr spec with placement count:2 2024-09-17T01:20:35.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:35 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:37.078 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:36 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/949280077' entity='client.admin' 2024-09-17T01:20:37.897 INFO:teuthology.orchestra.run.smithi081.stderr:Enabling the dashboard module... 2024-09-17T01:20:38.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:38 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2882852142' entity='client.admin' 2024-09-17T01:20:38.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:38 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:38.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:38 smithi081 ceph-mon[23696]: from='mgr.14120 172.21.15.81:0/1868126550' entity='mgr.y' 2024-09-17T01:20:39.338 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1766772670' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T01:20:39.339 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setuser ceph since I am not root 2024-09-17T01:20:39.339 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setgroup ceph since I am not root 2024-09-17T01:20:39.339 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.228+0000 7f08d87a9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:20:39.339 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.307+0000 7f08d87a9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:39.605 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.482+0000 7f08d87a9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:20:39.605 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.569+0000 7f08d87a9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:20:39.921 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.704+0000 7f08d87a9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:39.921 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:39.921+0000 7f08d87a9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:20:40.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:40 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1766772670' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T01:20:40.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:40 smithi081 ceph-mon[23696]: mgrmap e9: y(active, since 15s) 2024-09-17T01:20:40.270 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:40.270 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T01:20:40.270 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "available": true, 2024-09-17T01:20:40.270 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-17T01:20:40.271 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T01:20:40.271 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:40.523 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:40 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:40.264+0000 7f08d87a9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:20:40.523 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:40 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:40.357+0000 7f08d87a9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:20:40.524 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:40 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:40.524+0000 7f08d87a9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:20:40.848 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for the mgr to restart... 2024-09-17T01:20:40.848 INFO:teuthology.orchestra.run.smithi081.stderr:Waiting for mgr epoch 9... 2024-09-17T01:20:41.173 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:41 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2216871400' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T01:20:41.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:41.587+0000 7f08d87a9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:20:41.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:41.676+0000 7f08d87a9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:20:42.863 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:42.561+0000 7f08d87a9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:20:43.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:42.864+0000 7f08d87a9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:20:43.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:42.996+0000 7f08d87a9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:20:43.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.083+0000 7f08d87a9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:20:43.506 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.181+0000 7f08d87a9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:20:43.506 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.268+0000 7f08d87a9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:20:43.506 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.414+0000 7f08d87a9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:20:43.506 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.506+0000 7f08d87a9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:20:43.882 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.603+0000 7f08d87a9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:20:44.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.883+0000 7f08d87a9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:20:44.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:43.971+0000 7f08d87a9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:20:44.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:44.059+0000 7f08d87a9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:20:45.404 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:20:45.114+0000 7f08d87a9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:20:46.374 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: Active manager daemon y restarted 2024-09-17T01:20:46.374 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: Activating manager daemon y 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: osdmap e3: 0 total, 0 up, 0 in 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: mgrmap e10: y(active, starting, since 0.819384s) 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:20:46.375 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:20:46.376 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:20:46.945 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: { 2024-09-17T01:20:46.945 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T01:20:46.945 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T01:20:46.945 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: } 2024-09-17T01:20:46.957 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:46] ENGINE Bus STARTING 2024-09-17T01:20:46.957 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:46] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:20:46.957 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:20:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:20:46] ENGINE Bus STARTED 2024-09-17T01:20:47.550 INFO:teuthology.orchestra.run.smithi081.stderr:mgr epoch 9 is available 2024-09-17T01:20:47.550 INFO:teuthology.orchestra.run.smithi081.stderr:Generating a dashboard self-signed certificate... 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:46] ENGINE Bus STARTING 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:46] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: [17/Sep/2024:01:20:46] ENGINE Bus STARTED 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: mgrmap e11: y(active, since 1.82443s) 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T01:20:48.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:48.317 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T01:20:48.942 INFO:teuthology.orchestra.run.smithi081.stderr:Creating initial admin user... 2024-09-17T01:20:49.057 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:48 smithi081 ceph-mon[23696]: mgrmap e12: y(active, since 2s) 2024-09-17T01:20:49.057 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:48 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:49.057 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:48 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:49.057 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:48 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:50.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:49 smithi081 ceph-mon[23696]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:50.469 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$WqR3v.vltcMavQE1pHObyulkV9dL58i1AtAC5qm8ZpAgEmdY/S9sS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726536050, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T01:20:51.075 INFO:teuthology.orchestra.run.smithi081.stderr:Fetching dashboard port number... 2024-09-17T01:20:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:51 smithi081 ceph-mon[23696]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:20:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:51.697 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: 8443 2024-09-17T01:20:52.295 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: disabled 2024-09-17T01:20:52.303 INFO:teuthology.orchestra.run.smithi081.stderr:systemctl: inactive 2024-09-17T01:20:52.304 INFO:teuthology.orchestra.run.smithi081.stderr:firewalld.service is not enabled 2024-09-17T01:20:52.304 INFO:teuthology.orchestra.run.smithi081.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr:Ceph Dashboard is now available at: 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr: URL: https://smithi081.front.sepia.ceph.com:8443/ 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr: User: admin 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr: Password: 0xipdxmww0 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:52.306 INFO:teuthology.orchestra.run.smithi081.stderr:Enabling autotune for osd_memory_target 2024-09-17T01:20:52.552 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:52 smithi081 ceph-mon[23696]: mgrmap e13: y(active, since 6s) 2024-09-17T01:20:52.552 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:52 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/971681116' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T01:20:54.234 INFO:teuthology.orchestra.run.smithi081.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-17T01:20:54.814 INFO:teuthology.orchestra.run.smithi081.stderr:You can access the Ceph CLI with: 2024-09-17T01:20:54.814 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.814 INFO:teuthology.orchestra.run.smithi081.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: ceph telemetry on 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr:For more information see: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T01:20:54.815 INFO:teuthology.orchestra.run.smithi081.stderr:Bootstrap complete. 2024-09-17T01:20:54.849 INFO:tasks.cephadm:Fetching config... 2024-09-17T01:20:54.849 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:20:54.849 DEBUG:teuthology.orchestra.run.smithi081:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T01:20:54.866 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T01:20:54.866 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:20:54.866 DEBUG:teuthology.orchestra.run.smithi081:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T01:20:54.921 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T01:20:54.922 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:20:54.922 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd if=/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-17T01:20:54.986 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T01:20:54.987 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:20:54.987 DEBUG:teuthology.orchestra.run.smithi081:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T01:20:55.042 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T01:20:55.043 DEBUG:teuthology.orchestra.run.smithi081:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFYqJZL6DIkghT/W76axRvXIxjV5jSaKa6bcHlj+UB6dtxNFNLRE3/2BGSoWDDzF817MDPZ3g1OnezOTwLoflnOOH/H73GoRSXgDgA0taN4oDfgDQIPJMxxe6/tt8VnyKo+TWbOXW3KYr69Kb+CV/BWfR1BNynJHyPMpYBh3YHQTUgRNc5KhBtCU/EKJrZG3iWIaPkBUJx8GroLWlQKakwLcmvwrHVC0EcyWUKl13bCa2GMiPuRf2zCCfm4MYtJL7HZpVucTs/z6r3Ehvi2EChf3Wb5Jczu53694iqQaOS9Ae3hSrxF7EDBKtz4ah9aN0mmfm1LdsHEL8TGPWwdVfQyJNHkqH8jSFlPdUIFLYiNwCTkuSKEk3TRdbdng+VP+DnjNQvFZq8MjnNxuBxwHhiLkR0WFEFQesGBagWXxhZ1zf9D3+BrmI6/4Yvos8jogspE5mjZ9KHbV39P1nztumWyoYS4TtqJdY7WTDz00WPF23+5rNpHrCTI7VMXMqbYDE= ceph-ded6f264-7492-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T01:20:55.116 INFO:teuthology.orchestra.run.smithi081.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFYqJZL6DIkghT/W76axRvXIxjV5jSaKa6bcHlj+UB6dtxNFNLRE3/2BGSoWDDzF817MDPZ3g1OnezOTwLoflnOOH/H73GoRSXgDgA0taN4oDfgDQIPJMxxe6/tt8VnyKo+TWbOXW3KYr69Kb+CV/BWfR1BNynJHyPMpYBh3YHQTUgRNc5KhBtCU/EKJrZG3iWIaPkBUJx8GroLWlQKakwLcmvwrHVC0EcyWUKl13bCa2GMiPuRf2zCCfm4MYtJL7HZpVucTs/z6r3Ehvi2EChf3Wb5Jczu53694iqQaOS9Ae3hSrxF7EDBKtz4ah9aN0mmfm1LdsHEL8TGPWwdVfQyJNHkqH8jSFlPdUIFLYiNwCTkuSKEk3TRdbdng+VP+DnjNQvFZq8MjnNxuBxwHhiLkR0WFEFQesGBagWXxhZ1zf9D3+BrmI6/4Yvos8jogspE5mjZ9KHbV39P1nztumWyoYS4TtqJdY7WTDz00WPF23+5rNpHrCTI7VMXMqbYDE= ceph-ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:20:55.127 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFYqJZL6DIkghT/W76axRvXIxjV5jSaKa6bcHlj+UB6dtxNFNLRE3/2BGSoWDDzF817MDPZ3g1OnezOTwLoflnOOH/H73GoRSXgDgA0taN4oDfgDQIPJMxxe6/tt8VnyKo+TWbOXW3KYr69Kb+CV/BWfR1BNynJHyPMpYBh3YHQTUgRNc5KhBtCU/EKJrZG3iWIaPkBUJx8GroLWlQKakwLcmvwrHVC0EcyWUKl13bCa2GMiPuRf2zCCfm4MYtJL7HZpVucTs/z6r3Ehvi2EChf3Wb5Jczu53694iqQaOS9Ae3hSrxF7EDBKtz4ah9aN0mmfm1LdsHEL8TGPWwdVfQyJNHkqH8jSFlPdUIFLYiNwCTkuSKEk3TRdbdng+VP+DnjNQvFZq8MjnNxuBxwHhiLkR0WFEFQesGBagWXxhZ1zf9D3+BrmI6/4Yvos8jogspE5mjZ9KHbV39P1nztumWyoYS4TtqJdY7WTDz00WPF23+5rNpHrCTI7VMXMqbYDE= ceph-ded6f264-7492-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T01:20:55.179 INFO:teuthology.orchestra.run.smithi105.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFYqJZL6DIkghT/W76axRvXIxjV5jSaKa6bcHlj+UB6dtxNFNLRE3/2BGSoWDDzF817MDPZ3g1OnezOTwLoflnOOH/H73GoRSXgDgA0taN4oDfgDQIPJMxxe6/tt8VnyKo+TWbOXW3KYr69Kb+CV/BWfR1BNynJHyPMpYBh3YHQTUgRNc5KhBtCU/EKJrZG3iWIaPkBUJx8GroLWlQKakwLcmvwrHVC0EcyWUKl13bCa2GMiPuRf2zCCfm4MYtJL7HZpVucTs/z6r3Ehvi2EChf3Wb5Jczu53694iqQaOS9Ae3hSrxF7EDBKtz4ah9aN0mmfm1LdsHEL8TGPWwdVfQyJNHkqH8jSFlPdUIFLYiNwCTkuSKEk3TRdbdng+VP+DnjNQvFZq8MjnNxuBxwHhiLkR0WFEFQesGBagWXxhZ1zf9D3+BrmI6/4Yvos8jogspE5mjZ9KHbV39P1nztumWyoYS4TtqJdY7WTDz00WPF23+5rNpHrCTI7VMXMqbYDE= ceph-ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:20:55.192 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-17T01:20:55.373 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:55 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2592971447' entity='client.admin' 2024-09-17T01:20:57.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:57 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:57.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:57 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi081", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:20:57.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:57 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:57.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:57 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:57.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:57 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:20:58.233 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-17T01:20:58.233 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-17T01:20:58.782 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:58 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1883170043' entity='client.admin' 2024-09-17T01:20:58.782 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:58 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:20:58.782 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:20:58 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:00.836 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi105 2024-09-17T01:21:00.836 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:21:00.836 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.conf 2024-09-17T01:21:00.855 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:21:00.855 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:00.914 INFO:tasks.cephadm:Adding host smithi105 to orchestrator... 2024-09-17T01:21:00.914 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch host add smithi105 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:01.382 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:21:01.383 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:01.383 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:02.346 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:02 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:03.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:03 smithi081 ceph-mon[23696]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:03.664 INFO:teuthology.orchestra.run.smithi081.stdout:Added host 'smithi105' with addr '172.21.15.105' 2024-09-17T01:21:04.257 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: Deploying cephadm binary to smithi105 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: Added host smithi105 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:04.872 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:06.137 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:21:06.137 INFO:teuthology.orchestra.run.smithi081.stdout:[{"addr": "172.21.15.81", "hostname": "smithi081", "labels": [], "status": ""}, {"addr": "172.21.15.105", "hostname": "smithi105", "labels": [], "status": ""}] 2024-09-17T01:21:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:06 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:06 smithi081 ceph-mon[23696]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:06.711 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T01:21:06.711 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T01:21:07.349 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:07 smithi081 ceph-mon[23696]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:21:08.860 INFO:teuthology.orchestra.run.smithi081.stderr:adjusted tunables profile to default 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:09.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/662948819' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T01:21:09.407 INFO:tasks.cephadm:Adding mon.a on smithi081 2024-09-17T01:21:09.407 INFO:tasks.cephadm:Adding mon.c on smithi081 2024-09-17T01:21:09.407 INFO:tasks.cephadm:Adding mon.b on smithi105 2024-09-17T01:21:09.407 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b' 2024-09-17T01:21:10.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:09 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/662948819' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T01:21:10.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:09 smithi081 ceph-mon[23696]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:11.302 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mon update... 2024-09-17T01:21:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:10 smithi081 ceph-mon[23696]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:12.011 DEBUG:teuthology.orchestra.run.smithi081:mon.c> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.c.service 2024-09-17T01:21:12.013 DEBUG:teuthology.orchestra.run.smithi105:mon.b> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.b.service 2024-09-17T01:21:12.015 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T01:21:12.015 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T01:21:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: Saving service mon spec with placement smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:12.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: Deploying daemon mon.c on smithi081 2024-09-17T01:21:12.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:12 smithi081 ceph-mon[23696]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:13.717 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:21:13.718 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":1,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","modified":"2024-09-17T01:19:43.267953Z","created":"2024-09-17T01:19:43.267953Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3300","nonce":0},{"type":"v1","addr":"172.21.15.81:6789","nonce":0}]},"addr":"172.21.15.81:6789/0","public_addr":"172.21.15.81:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T01:21:13.719 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 1 2024-09-17T01:21:13.831 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:13 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/3151527458' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:15.239 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T01:21:15.239 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T01:21:20.125 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:21:20.125 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":2,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","modified":"2024-09-17T01:21:14.661174Z","created":"2024-09-17T01:19:43.267953Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3300","nonce":0},{"type":"v1","addr":"172.21.15.81:6789","nonce":0}]},"addr":"172.21.15.81:6789/0","public_addr":"172.21.15.81:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3301","nonce":0},{"type":"v1","addr":"172.21.15.81:6790","nonce":0}]},"addr":"172.21.15.81:6790/0","public_addr":"172.21.15.81:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T01:21:20.127 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 2 2024-09-17T01:21:20.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:20.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: mon.a calling monitor election 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: mon.c calling monitor election 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: monmap e2: 2 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:21:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: fsmap 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: mgrmap e13: y(active, since 34s) 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: overall HEALTH_OK 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:20.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[23696]: Deploying daemon mon.b on smithi105 2024-09-17T01:21:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[23696]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/386884118' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:21.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[28146]: Deploying daemon mon.b on smithi105 2024-09-17T01:21:21.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[28146]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:21.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/386884118' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:21.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:20 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:22.351 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T01:21:22.352 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T01:21:23.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:22 smithi081 ceph-mon[23696]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:22 smithi081 ceph-mon[28146]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.498 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:21:24.498 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":2,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","modified":"2024-09-17T01:21:14.661174Z","created":"2024-09-17T01:19:43.267953Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3300","nonce":0},{"type":"v1","addr":"172.21.15.81:6789","nonce":0}]},"addr":"172.21.15.81:6789/0","public_addr":"172.21.15.81:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3301","nonce":0},{"type":"v1","addr":"172.21.15.81:6790","nonce":0}]},"addr":"172.21.15.81:6790/0","public_addr":"172.21.15.81:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T01:21:24.499 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 2 2024-09-17T01:21:24.627 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 podman[24569]: 2024-09-17 01:21:24.625676411 +0000 UTC m=+0.341348693 container init 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, release=754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, distribution-scope=public, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8) 2024-09-17T01:21:24.627 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 podman[24569]: 2024-09-17 01:21:24.629320143 +0000 UTC m=+0.344992412 container start 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pidfile_write: ignore empty --pid-file 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: load: jerasure load: lrc 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: RocksDB version: 6.15.5 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Compile date Apr 18 2022 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: DB SUMMARY 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: DB Session ID: SXCGIZC5QSW4EIX7VEA8 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:21:24.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.env: 0x55721aec6860 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.fs: Posix File System 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.info_log: 0x55721d235dc0 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.statistics: (nil) 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.use_fsync: 0 2024-09-17T01:21:24.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.db_log_dir: 2024-09-17T01:21:24.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-17T01:21:24.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:21:24.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:21:24.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:21:24.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:21:24.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:21:24.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:21:24.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:21:24.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.write_buffer_manager: 0x55721d326240 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.unordered_write: 0 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:21:24.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.row_cache: None 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.wal_filter: None 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.preserve_deletes: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:21:24.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:21:24.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_open_files: -1 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Compression algorithms supported: 2024-09-17T01:21:24.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kZSTD supported: 0 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.merge_operator: 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_filter: None 2024-09-17T01:21:24.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55721d201d00) 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_type: 0 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_index_type: 0 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_shortening: 1 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: hash_index_allow_collision: 1 2024-09-17T01:21:24.896 INFO:journalctl@ceph.mon.b.smithi105.stdout: checksum: 1 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: no_block_cache: 0 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache: 0x55721d26c170 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_options: 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: capacity : 536870912 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_shard_bits : 4 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: strict_capacity_limit : 0 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_compressed: (nil) 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: persistent_cache: (nil) 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size: 4096 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size_deviation: 10 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_restart_interval: 16 2024-09-17T01:21:24.897 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_block_restart_interval: 1 2024-09-17T01:21:24.898 INFO:journalctl@ceph.mon.b.smithi105.stdout: metadata_block_size: 4096 2024-09-17T01:21:24.898 INFO:journalctl@ceph.mon.b.smithi105.stdout: partition_filters: 0 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: use_delta_encoding: 1 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: whole_key_filtering: 1 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: verify_compression: 0 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:21:24.900 INFO:journalctl@ceph.mon.b.smithi105.stdout: format_version: 4 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_index_compression: 1 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_align: 0 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression: NoCompression 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.num_levels: 7 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:21:24.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:21:24.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:21:24.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:21:24.904 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.table_properties_collectors: 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:21:24.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.ttl: 2592000 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:21:24.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726536084656236, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726536084656533, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726536084, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e46a32cb-03d4-4d6a-b644-a22d423e195e", "db_session_id": "SXCGIZC5QSW4EIX7VEA8"}} 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726536084656836, "job": 1, "event": "recovery_finished"} 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55721d252a80 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: DB pointer 0x55721d262000 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T01:21:24.907 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** DB Stats ** 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:21:24.908 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:21:24.909 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:21:24.910 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:21:24.911 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: using public_addr v2:172.21.15.105:0/0 -> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: starting mon.b rank -1 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(???) e0 preinit fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).mds e1 new map 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: e1 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: legacy client fscid: -1 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout: No filesystems configured 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-17T01:21:24.912 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: Saving service mon spec with placement smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:24.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: Deploying daemon mon.c on smithi081 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/3151527458' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.a calling monitor election 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.c calling monitor election 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: monmap e2: 2 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: fsmap 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mgrmap e13: y(active, since 34s) 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: overall HEALTH_OK 2024-09-17T01:21:24.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: Deploying daemon mon.b on smithi105 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/386884118' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing).paxosservice(auth 1..4) refresh upgraded, format 0 -> 3 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 ceph-mon[24602]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 bash[24569]: 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 2024-09-17T01:21:24.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:24 smithi105 systemd[1]: Started Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:21:26.307 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T01:21:26.307 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T01:21:30.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: mon.a calling monitor election 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: mon.c calling monitor election 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: mon.b calling monitor election 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: fsmap 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: mgrmap e13: y(active, since 44s) 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: overall HEALTH_OK 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:29 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:21:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: mon.a calling monitor election 2024-09-17T01:21:30.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: mon.c calling monitor election 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: mon.b calling monitor election 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:21:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: fsmap 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: mgrmap e13: y(active, since 44s) 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: overall HEALTH_OK 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: mon.a calling monitor election 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: mon.c calling monitor election 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: mon.b calling monitor election 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: fsmap 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: mgrmap e13: y(active, since 44s) 2024-09-17T01:21:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: overall HEALTH_OK 2024-09-17T01:21:30.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:30.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:30.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:30.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:29 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:31.198 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:21:31.198 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":3,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","modified":"2024-09-17T01:21:24.676494Z","created":"2024-09-17T01:19:43.267953Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3300","nonce":0},{"type":"v1","addr":"172.21.15.81:6789","nonce":0}]},"addr":"172.21.15.81:6789/0","public_addr":"172.21.15.81:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:3301","nonce":0},{"type":"v1","addr":"172.21.15.81:6790","nonce":0}]},"addr":"172.21.15.81:6790/0","public_addr":"172.21.15.81:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-17T01:21:31.200 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 3 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:31.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:31.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:30 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:31.231 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:31.231 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.231 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.231 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:21:31.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.233 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.233 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:31.233 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:31.233 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:31.233 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:30 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:31.747 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T01:21:31.748 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T01:21:32.050 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: Reconfiguring mon.a (unknown last config time)... 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/307033526' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:21:32.051 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: Reconfiguring mon.a (unknown last config time)... 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/307033526' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:32.052 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:21:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:21:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: Reconfiguring mon.a (unknown last config time)... 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/307033526' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:32.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[23696]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:21:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[23696]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:21:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[23696]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:33.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[28146]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:21:33.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[28146]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:21:33.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:32 smithi081 ceph-mon[28146]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:32 smithi105 ceph-mon[24602]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:21:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:32 smithi105 ceph-mon[24602]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:21:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:32 smithi105 ceph-mon[24602]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:34.792 INFO:teuthology.orchestra.run.smithi081.stdout:# minimal ceph.conf for ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:21:34.793 INFO:teuthology.orchestra.run.smithi081.stdout:[global] 2024-09-17T01:21:34.793 INFO:teuthology.orchestra.run.smithi081.stdout: fsid = ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:21:34.793 INFO:teuthology.orchestra.run.smithi081.stdout: mon_host = [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] 2024-09-17T01:21:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:35.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:35.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:35.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:34 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/71156609' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/71156609' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:35.311 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:21:35.312 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:21:35.312 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.312 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:34 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/71156609' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:35.412 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T01:21:35.412 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:21:35.413 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T01:21:35.443 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:21:35.443 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:35.508 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:21:35.508 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T01:21:35.534 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:21:35.534 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:35.599 INFO:tasks.cephadm:Adding mgr.y on smithi081 2024-09-17T01:21:35.599 INFO:tasks.cephadm:Adding mgr.x on smithi105 2024-09-17T01:21:35.600 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi081=y;smithi105=x' 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:36.193 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:37.076 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:36 smithi105 ceph-mon[24602]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:36 smithi081 ceph-mon[23696]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:37.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:36 smithi081 ceph-mon[28146]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:37.465 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mgr update... 2024-09-17T01:21:38.141 DEBUG:teuthology.orchestra.run.smithi105:mgr.x> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service 2024-09-17T01:21:38.143 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T01:21:38.143 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:21:38.144 DEBUG:teuthology.orchestra.run.smithi081:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T01:21:38.164 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T01:21:38.164 DEBUG:teuthology.orchestra.run.smithi081:> stat /dev/vg_nvme/lv_1 2024-09-17T01:21:38.222 INFO:teuthology.orchestra.run.smithi081.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Access: 2024-09-17 01:21:14.139316410 +0000 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Modify: 2024-09-17 01:19:00.665333410 +0000 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout:Change: 2024-09-17 01:19:00.665333410 +0000 2024-09-17T01:21:38.223 INFO:teuthology.orchestra.run.smithi081.stdout: Birth: 2024-09-17 01:19:00.665333410 +0000 2024-09-17T01:21:38.223 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T01:21:38.291 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records in 2024-09-17T01:21:38.291 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records out 2024-09-17T01:21:38.291 INFO:teuthology.orchestra.run.smithi081.stderr:512 bytes copied, 0.000234873 s, 2.2 MB/s 2024-09-17T01:21:38.293 DEBUG:teuthology.orchestra.run.smithi081:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T01:21:38.352 DEBUG:teuthology.orchestra.run.smithi081:> stat /dev/vg_nvme/lv_2 2024-09-17T01:21:38.409 INFO:teuthology.orchestra.run.smithi081.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T01:21:38.410 INFO:teuthology.orchestra.run.smithi081.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:38.410 INFO:teuthology.orchestra.run.smithi081.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-17T01:21:38.410 INFO:teuthology.orchestra.run.smithi081.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:38.410 INFO:teuthology.orchestra.run.smithi081.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:38.410 INFO:teuthology.orchestra.run.smithi081.stdout:Access: 2024-09-17 01:21:14.139316410 +0000 2024-09-17T01:21:38.411 INFO:teuthology.orchestra.run.smithi081.stdout:Modify: 2024-09-17 01:19:00.979327507 +0000 2024-09-17T01:21:38.411 INFO:teuthology.orchestra.run.smithi081.stdout:Change: 2024-09-17 01:19:00.979327507 +0000 2024-09-17T01:21:38.411 INFO:teuthology.orchestra.run.smithi081.stdout: Birth: 2024-09-17 01:19:00.979327507 +0000 2024-09-17T01:21:38.411 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T01:21:38.475 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records in 2024-09-17T01:21:38.476 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records out 2024-09-17T01:21:38.476 INFO:teuthology.orchestra.run.smithi081.stderr:512 bytes copied, 0.000192403 s, 2.7 MB/s 2024-09-17T01:21:38.477 DEBUG:teuthology.orchestra.run.smithi081:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T01:21:38.534 DEBUG:teuthology.orchestra.run.smithi081:> stat /dev/vg_nvme/lv_3 2024-09-17T01:21:38.591 INFO:teuthology.orchestra.run.smithi081.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Access: 2024-09-17 01:21:14.139316410 +0000 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Modify: 2024-09-17 01:19:01.300321473 +0000 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout:Change: 2024-09-17 01:19:01.300321473 +0000 2024-09-17T01:21:38.592 INFO:teuthology.orchestra.run.smithi081.stdout: Birth: 2024-09-17 01:19:01.300321473 +0000 2024-09-17T01:21:38.592 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T01:21:38.655 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi081=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:38.655 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: Saving service mgr spec with placement smithi081=y;smithi105=x;count:2 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T01:21:38.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: Deploying daemon mgr.x on smithi105 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[23696]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi081=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: Saving service mgr spec with placement smithi081=y;smithi105=x;count:2 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:38.657 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: Deploying daemon mgr.x on smithi105 2024-09-17T01:21:38.658 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:38 smithi081 ceph-mon[28146]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:38.659 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records in 2024-09-17T01:21:38.660 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records out 2024-09-17T01:21:38.660 INFO:teuthology.orchestra.run.smithi081.stderr:512 bytes copied, 0.000195733 s, 2.6 MB/s 2024-09-17T01:21:38.660 DEBUG:teuthology.orchestra.run.smithi081:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T01:21:38.717 DEBUG:teuthology.orchestra.run.smithi081:> stat /dev/vg_nvme/lv_4 2024-09-17T01:21:38.773 INFO:teuthology.orchestra.run.smithi081.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Access: 2024-09-17 01:21:14.140316391 +0000 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Modify: 2024-09-17 01:19:01.625315364 +0000 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout:Change: 2024-09-17 01:19:01.625315364 +0000 2024-09-17T01:21:38.774 INFO:teuthology.orchestra.run.smithi081.stdout: Birth: 2024-09-17 01:19:01.625315364 +0000 2024-09-17T01:21:38.774 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T01:21:38.839 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records in 2024-09-17T01:21:38.839 INFO:teuthology.orchestra.run.smithi081.stderr:1+0 records out 2024-09-17T01:21:38.839 INFO:teuthology.orchestra.run.smithi081.stderr:512 bytes copied, 0.000258456 s, 2.0 MB/s 2024-09-17T01:21:38.840 DEBUG:teuthology.orchestra.run.smithi081:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T01:21:38.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi081=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:38.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: Saving service mgr spec with placement smithi081=y;smithi105=x;count:2 2024-09-17T01:21:38.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: Deploying daemon mgr.x on smithi105 2024-09-17T01:21:38.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:38 smithi105 ceph-mon[24602]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:38.898 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:21:38.898 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T01:21:38.914 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T01:21:38.915 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-09-17T01:21:38.970 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T01:21:38.970 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:38.970 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-17T01:21:38.970 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:38.970 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:38.971 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 01:21:23.299689211 +0000 2024-09-17T01:21:38.971 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 01:19:00.673887809 +0000 2024-09-17T01:21:38.971 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 01:19:00.673887809 +0000 2024-09-17T01:21:38.971 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 01:19:00.673887809 +0000 2024-09-17T01:21:38.971 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T01:21:39.035 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T01:21:39.035 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T01:21:39.035 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000208253 s, 2.5 MB/s 2024-09-17T01:21:39.036 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T01:21:39.093 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:39.149 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 01:21:23.300689190 +0000 2024-09-17T01:21:39.150 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 01:19:01.001880976 +0000 2024-09-17T01:21:39.150 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 01:19:01.001880976 +0000 2024-09-17T01:21:39.150 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 01:19:01.001880976 +0000 2024-09-17T01:21:39.150 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T01:21:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T01:21:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T01:21:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000199073 s, 2.6 MB/s 2024-09-17T01:21:39.216 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T01:21:39.272 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 01:21:23.300689190 +0000 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 01:19:01.332874080 +0000 2024-09-17T01:21:39.329 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 01:19:01.332874080 +0000 2024-09-17T01:21:39.330 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 01:19:01.332874080 +0000 2024-09-17T01:21:39.330 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T01:21:39.420 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T01:21:39.420 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T01:21:39.420 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000192143 s, 2.7 MB/s 2024-09-17T01:21:39.421 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T01:21:39.437 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-09-17T01:21:39.497 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T01:21:39.497 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 01:21:39.381354662 +0000 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 01:19:01.671867018 +0000 2024-09-17T01:21:39.498 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 01:19:01.671867018 +0000 2024-09-17T01:21:39.499 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 01:19:01.671867018 +0000 2024-09-17T01:21:39.499 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T01:21:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T01:21:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T01:21:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000175237 s, 2.9 MB/s 2024-09-17T01:21:39.596 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T01:21:39.612 INFO:tasks.cephadm:Deploying osd.0 on smithi081 with /dev/vg_nvme/lv_4... 2024-09-17T01:21:39.612 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T01:21:39.823 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:39 smithi105 systemd[1]: Starting Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:21:40.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:39 smithi105 podman[25755]: 2024-09-17 01:21:39.728774621 +0000 UTC m=+0.016141692 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-17T01:21:40.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:39 smithi105 podman[25755]: 2024-09-17 01:21:39.854572606 +0000 UTC m=+0.141939659 container create faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T01:21:40.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 podman[25755]: 2024-09-17 01:21:40.048616761 +0000 UTC m=+0.335983826 container init faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., name=centos-stream) 2024-09-17T01:21:40.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 podman[25755]: 2024-09-17 01:21:40.052081167 +0000 UTC m=+0.339448222 container start faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD) 2024-09-17T01:21:40.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mgr[25775]: -- 172.21.15.105:0/2768694089 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5590311a6340 con 0x55903117f000 2024-09-17T01:21:40.465 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 bash[25755]: faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 2024-09-17T01:21:40.466 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 systemd[1]: Started Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:21:40.466 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.193+0000 7f5a2570f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:21:40.466 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.276+0000 7f5a2570f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:21:40.466 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.467+0000 7f5a2570f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:21:40.776 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.572+0000 7f5a2570f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:21:40.777 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.706+0000 7f5a2570f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:21:41.032 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:40.922+0000 7f5a2570f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:21:41.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mon[24602]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:41.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:41.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:41.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:41.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:40 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:41.098 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[23696]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[28146]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:41.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:40 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:41.282 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:41.255+0000 7f5a2570f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:21:41.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:41.346+0000 7f5a2570f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:21:41.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:41.506+0000 7f5a2570f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:21:42.131 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:21:42.148 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi081:vg_nvme/lv_4 2024-09-17T01:21:42.299 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.299 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:42.300 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:42.301 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:41 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:21:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:21:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:41 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:42.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:42.537+0000 7f5a2570f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:21:42.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:42.625+0000 7f5a2570f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:21:43.267 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[23696]: Reconfiguring mgr.y (unknown last config time)... 2024-09-17T01:21:43.267 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[23696]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:21:43.267 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[23696]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:43.267 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[28146]: Reconfiguring mgr.y (unknown last config time)... 2024-09-17T01:21:43.268 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[28146]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:21:43.268 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:42 smithi081 ceph-mon[28146]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:42 smithi105 ceph-mon[24602]: Reconfiguring mgr.y (unknown last config time)... 2024-09-17T01:21:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:42 smithi105 ceph-mon[24602]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:21:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:42 smithi105 ceph-mon[24602]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:43.810 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:43.511+0000 7f5a2570f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:21:44.121 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:43.811+0000 7f5a2570f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:21:44.122 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:43.943+0000 7f5a2570f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:21:44.122 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.028+0000 7f5a2570f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:21:44.122 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.123+0000 7f5a2570f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:21:44.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.210+0000 7f5a2570f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:21:44.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.354+0000 7f5a2570f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:21:44.815 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.444+0000 7f5a2570f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:21:44.815 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.538+0000 7f5a2570f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:21:45.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.817+0000 7f5a2570f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:21:45.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.904+0000 7f5a2570f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:44.991+0000 7f5a2570f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:21:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:21:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:21:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:21:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:21:45.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:46.294 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:45 smithi081 ceph-mon[23696]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:46.294 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:45 smithi081 ceph-mon[28146]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:45 smithi105 ceph-mon[24602]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:21:46.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:21:46.048+0000 7f5a2570f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:21:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:21:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:21:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:21:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:21:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:21:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:21:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:46 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:21:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:21:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:21:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:21:47.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:21:47.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:46 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1909309991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:21:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]: dispatch 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]': finished 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: osdmap e5: 1 total, 0 up, 1 in 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:21:48.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2344710574' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:21:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-17T01:21:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:21:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]: dispatch 2024-09-17T01:21:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]': finished 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: osdmap e5: 1 total, 0 up, 1 in 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2344710574' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: mgrmap e14: y(active, since 61s), standbys: x 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]: dispatch 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1332561062' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfcee6db-e2f0-4f9f-bae8-a55f4671b506"}]': finished 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: osdmap e5: 1 total, 0 up, 1 in 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:21:48.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2344710574' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:21:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:48 smithi105 ceph-mon[24602]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:49.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:48 smithi081 ceph-mon[23696]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:48 smithi081 ceph-mon[28146]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:51.133 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:50 smithi081 ceph-mon[23696]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:51.133 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:50 smithi081 ceph-mon[28146]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:50 smithi105 ceph-mon[24602]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:52.952 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:52 smithi081 ceph-mon[23696]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:52.953 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:52 smithi081 ceph-mon[28146]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:52 smithi105 ceph-mon[24602]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:54.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:53 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:21:54.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:53 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:54.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:53 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:21:54.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:53 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:54.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:53 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:21:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:53 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:54 smithi081 ceph-mon[23696]: Deploying daemon osd.0 on smithi081 2024-09-17T01:21:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:54 smithi081 ceph-mon[23696]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:55.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:54 smithi081 ceph-mon[28146]: Deploying daemon osd.0 on smithi081 2024-09-17T01:21:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:54 smithi081 ceph-mon[28146]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:55.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:54 smithi105 ceph-mon[24602]: Deploying daemon osd.0 on smithi081 2024-09-17T01:21:55.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:54 smithi105 ceph-mon[24602]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:57.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:56 smithi081 ceph-mon[23696]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:56 smithi081 ceph-mon[28146]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:56 smithi105 ceph-mon[24602]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:59 smithi105 ceph-mon[24602]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:21:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[23696]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:59.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:59.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:59.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:59.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:21:59.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[28146]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:21:59.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:21:59.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:21:59.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:21:59.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:21:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:01.227 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[23696]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:01.227 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.227 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.227 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[28146]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:01.227 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.228 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:00 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.257 INFO:teuthology.orchestra.run.smithi081.stdout:Created osd(s) 0 on host 'smithi081' 2024-09-17T01:22:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:00 smithi105 ceph-mon[24602]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:00 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:00 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:01.860 DEBUG:teuthology.orchestra.run.smithi081:osd.0> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T01:22:01.863 INFO:tasks.cephadm:Deploying osd.1 on smithi081 with /dev/vg_nvme/lv_3... 2024-09-17T01:22:01.863 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T01:22:02.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[28146]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:22:02.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:02.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:02.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:02.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[23696]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:22:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:01 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:01 smithi105 ceph-mon[24602]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:22:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:01 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:01 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:01 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:01 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:03.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[28146]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:03.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[28146]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:22:03.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[28146]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T01:22:03.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:03.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[28146]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:03.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[23696]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:03.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[23696]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:22:03.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[23696]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T01:22:03.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:03.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-mon[23696]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:03.182 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[30992]: 2024-09-17T01:22:02.965+0000 7f14a2e75700 -1 osd.0 0 waiting for initial osdmap 2024-09-17T01:22:03.182 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:02 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[30992]: 2024-09-17T01:22:02.969+0000 7f149d80c700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:22:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:02 smithi105 ceph-mon[24602]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:02 smithi105 ceph-mon[24602]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:22:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:02 smithi105 ceph-mon[24602]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T01:22:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:02 smithi105 ceph-mon[24602]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[23696]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[23696]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T01:22:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[28146]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[28146]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T01:22:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:03 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:03 smithi105 ceph-mon[24602]: from='osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:03 smithi105 ceph-mon[24602]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T01:22:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:03 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:03 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668] boot 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: osdmap e8: 1 total, 1 up, 1 in 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.243 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668] boot 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: osdmap e8: 1 total, 1 up, 1 in 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.244 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:04 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:22:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:22:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T01:22:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: osd.0 [v2:172.21.15.81:6802/2140831668,v1:172.21.15.81:6803/2140831668] boot 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: osdmap e8: 1 total, 1 up, 1 in 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:04 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi081 2024-09-17T01:22:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi081 to 16064M 2024-09-17T01:22:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[28146]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T01:22:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi081 2024-09-17T01:22:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi081 to 16064M 2024-09-17T01:22:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:05 smithi081 ceph-mon[23696]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T01:22:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:05 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi081 2024-09-17T01:22:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:05 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi081 to 16064M 2024-09-17T01:22:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:05 smithi105 ceph-mon[24602]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T01:22:06.454 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:22:06.472 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi081:vg_nvme/lv_3 2024-09-17T01:22:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:06 smithi105 ceph-mon[24602]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:06 smithi081 ceph-mon[23696]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:06 smithi081 ceph-mon[28146]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[23696]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:09.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[28146]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:09.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:09.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:09.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:08 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:08 smithi105 ceph-mon[24602]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:08 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:09.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:08 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:09.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:08 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:10 smithi105 ceph-mon[24602]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:10 smithi081 ceph-mon[23696]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:10.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:10 smithi081 ceph-mon[28146]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:11 smithi105 ceph-mon[24602]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]: dispatch 2024-09-17T01:22:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]': finished 2024-09-17T01:22:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:11 smithi105 ceph-mon[24602]: osdmap e10: 2 total, 1 up, 2 in 2024-09-17T01:22:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:11 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[28146]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]: dispatch 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]': finished 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[28146]: osdmap e10: 2 total, 1 up, 2 in 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[23696]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]: dispatch 2024-09-17T01:22:11.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/11729752' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59a4263a-67b4-4f17-8757-22aba74776a3"}]': finished 2024-09-17T01:22:11.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[23696]: osdmap e10: 2 total, 1 up, 2 in 2024-09-17T01:22:11.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:11 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:12.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:12 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3035442576' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:12 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3035442576' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:12.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:12 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3035442576' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:13.307 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:13 smithi081 ceph-mon[23696]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:13.307 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:13 smithi081 ceph-mon[28146]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:13 smithi105 ceph-mon[24602]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:15.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:15 smithi081 ceph-mon[28146]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:15.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:15 smithi081 ceph-mon[23696]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:15 smithi105 ceph-mon[24602]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[23696]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[28146]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:22:17.121 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:17 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:17 smithi105 ceph-mon[24602]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:17 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:22:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:17 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:18.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:18 smithi105 ceph-mon[24602]: Deploying daemon osd.1 on smithi081 2024-09-17T01:22:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:18 smithi081 ceph-mon[23696]: Deploying daemon osd.1 on smithi081 2024-09-17T01:22:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:18 smithi081 ceph-mon[28146]: Deploying daemon osd.1 on smithi081 2024-09-17T01:22:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:19 smithi081 ceph-mon[23696]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:19 smithi081 ceph-mon[28146]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:19.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:19 smithi105 ceph-mon[24602]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:21.206 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:20 smithi081 ceph-mon[23696]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:21.206 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:20 smithi081 ceph-mon[28146]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:21.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:20 smithi105 ceph-mon[24602]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:22.553 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:22.553 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:22.553 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:22.553 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:22.553 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[23696]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:22.554 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:22.554 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:22.554 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:22.554 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:22.554 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:22 smithi081 ceph-mon[28146]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:22 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:22 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:22 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:22.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:22 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:22.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:22 smithi105 ceph-mon[24602]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:24.080 INFO:teuthology.orchestra.run.smithi081.stdout:Created osd(s) 1 on host 'smithi081' 2024-09-17T01:22:24.634 DEBUG:teuthology.orchestra.run.smithi081:osd.1> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T01:22:24.636 INFO:tasks.cephadm:Deploying osd.2 on smithi081 with /dev/vg_nvme/lv_2... 2024-09-17T01:22:24.636 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T01:22:25.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[28146]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[28146]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[28146]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[23696]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:25.256 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[23696]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.257 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[23696]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.257 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:25 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:25.338 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:25 smithi105 ceph-mon[24602]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:25.338 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:25 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:25.338 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:25 smithi105 ceph-mon[24602]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.339 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:25 smithi105 ceph-mon[24602]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:22:25.339 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:25 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[23696]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[23696]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[23696]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[23696]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:26.119 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[28146]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:22:26.120 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[28146]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T01:22:26.120 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:26.120 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[28146]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:26.120 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-mon[28146]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:26.120 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[33929]: 2024-09-17T01:22:26.091+0000 7f85cf434700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T01:22:26.120 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[33929]: 2024-09-17T01:22:26.094+0000 7f85c9dcb700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:22:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:26 smithi105 ceph-mon[24602]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:22:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:26 smithi105 ceph-mon[24602]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T01:22:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:26.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:26 smithi105 ceph-mon[24602]: from='osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:26.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:26 smithi105 ceph-mon[24602]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:27 smithi105 ceph-mon[24602]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:27 smithi105 ceph-mon[24602]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:27 smithi105 ceph-mon[24602]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T01:22:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:27 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:27 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[23696]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[23696]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[23696]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[28146]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[28146]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[28146]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:27 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:27.541 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:22:27.562 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi081:vg_nvme/lv_2 2024-09-17T01:22:28.261 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:22:28.261 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[23696]: osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968] boot 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[23696]: osdmap e13: 2 total, 2 up, 2 in 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[28146]: osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968] boot 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[28146]: osdmap e13: 2 total, 2 up, 2 in 2024-09-17T01:22:28.262 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:28 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:28.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:28 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:22:28.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:28 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:22:28.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:28 smithi105 ceph-mon[24602]: osd.1 [v2:172.21.15.81:6810/2547106968,v1:172.21.15.81:6811/2547106968] boot 2024-09-17T01:22:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:28 smithi105 ceph-mon[24602]: osdmap e13: 2 total, 2 up, 2 in 2024-09-17T01:22:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:28 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:22:29.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:29 smithi105 ceph-mon[24602]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:29.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:29 smithi105 ceph-mon[24602]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T01:22:29.399 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:29 smithi081 ceph-mon[23696]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:29.399 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:29 smithi081 ceph-mon[23696]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T01:22:29.400 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:29 smithi081 ceph-mon[28146]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T01:22:29.400 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:29 smithi081 ceph-mon[28146]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T01:22:31.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:31.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:32.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:32 smithi105 ceph-mon[24602]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:32 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi081 2024-09-17T01:22:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:32 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi081 to 8032M 2024-09-17T01:22:32.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[28146]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:32.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi081 2024-09-17T01:22:32.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi081 to 8032M 2024-09-17T01:22:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[23696]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi081 2024-09-17T01:22:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:32 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi081 to 8032M 2024-09-17T01:22:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3225157691' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]': finished 2024-09-17T01:22:33.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: osdmap e15: 3 total, 2 up, 3 in 2024-09-17T01:22:33.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:33.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:33 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/716678883' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3225157691' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]': finished 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: osdmap e15: 3 total, 2 up, 3 in 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/716678883' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3225157691' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]: dispatch 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31cb2977-5e2e-462e-98b9-064f54a364c9"}]': finished 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: osdmap e15: 3 total, 2 up, 3 in 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:33 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/716678883' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:35 smithi105 ceph-mon[24602]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:35 smithi081 ceph-mon[23696]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:35.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:35 smithi081 ceph-mon[28146]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:37.129 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:37 smithi081 ceph-mon[23696]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:37.129 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:37 smithi081 ceph-mon[28146]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:37 smithi105 ceph-mon[24602]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:39 smithi105 ceph-mon[24602]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:22:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[23696]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:22:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[28146]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:22:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:40 smithi105 ceph-mon[24602]: Deploying daemon osd.2 on smithi081 2024-09-17T01:22:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:40 smithi081 ceph-mon[23696]: Deploying daemon osd.2 on smithi081 2024-09-17T01:22:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:40 smithi081 ceph-mon[28146]: Deploying daemon osd.2 on smithi081 2024-09-17T01:22:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:41 smithi105 ceph-mon[24602]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:41.394 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:41 smithi081 ceph-mon[23696]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:41.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:41 smithi081 ceph-mon[28146]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:43 smithi105 ceph-mon[24602]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:43 smithi081 ceph-mon[23696]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:43 smithi081 ceph-mon[28146]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:44 smithi105 ceph-mon[24602]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[23696]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:44 smithi081 ceph-mon[28146]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:46.500 INFO:teuthology.orchestra.run.smithi081.stdout:Created osd(s) 2 on host 'smithi081' 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:46.793 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[23696]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:46.794 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:46 smithi081 ceph-mon[28146]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:46.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:46 smithi105 ceph-mon[24602]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:22:47.052 DEBUG:teuthology.orchestra.run.smithi081:osd.2> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T01:22:47.054 INFO:tasks.cephadm:Deploying osd.3 on smithi081 with /dev/vg_nvme/lv_1... 2024-09-17T01:22:47.054 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T01:22:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:22:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T01:22:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T01:22:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:47.682 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[36841]: 2024-09-17T01:22:47.499+0000 7f4984e5b700 -1 osd.2 0 waiting for initial osdmap 2024-09-17T01:22:47.682 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[36841]: 2024-09-17T01:22:47.502+0000 7f497eff1700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:22:47.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:22:47.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T01:22:47.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:47.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:47 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[23696]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[23696]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[23696]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[28146]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[28146]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T01:22:48.531 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:48 smithi081 ceph-mon[28146]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:48 smithi105 ceph-mon[24602]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:22:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:48 smithi105 ceph-mon[24602]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T01:22:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:48 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:48 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:48 smithi105 ceph-mon[24602]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T01:22:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:22:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:22:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[23696]: osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537] boot 2024-09-17T01:22:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[23696]: osdmap e18: 3 total, 3 up, 3 in 2024-09-17T01:22:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:49.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:22:49.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:22:49.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[28146]: osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537] boot 2024-09-17T01:22:49.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[28146]: osdmap e18: 3 total, 3 up, 3 in 2024-09-17T01:22:49.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:49 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:49 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:22:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:49 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:22:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:49 smithi105 ceph-mon[24602]: osd.2 [v2:172.21.15.81:6818/2272030537,v1:172.21.15.81:6819/2272030537] boot 2024-09-17T01:22:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:49 smithi105 ceph-mon[24602]: osdmap e18: 3 total, 3 up, 3 in 2024-09-17T01:22:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:49 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi081 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi081 to 5354M 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:50 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T01:22:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T01:22:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi081 2024-09-17T01:22:50.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi081 to 5354M 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:50.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi081 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi081 to 5354M 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:50.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:50 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T01:22:51.524 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:22:51.544 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi081:vg_nvme/lv_1 2024-09-17T01:22:52.093 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T01:22:52.093 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[23696]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T01:22:52.093 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T01:22:52.094 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T01:22:52.094 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[28146]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T01:22:52.094 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T01:22:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T01:22:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:51 smithi105 ceph-mon[24602]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T01:22:52.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T01:22:52.846 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38129]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:52.846 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38129]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:52.846 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38129]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:53.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[23696]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:53.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T01:22:53.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[23696]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T01:22:53.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:53.138 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[28146]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:53.139 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T01:22:53.139 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[28146]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T01:22:53.139 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:53.139 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38150]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:53.139 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38150]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:53.139 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:22:52 smithi081 sudo[38150]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:52 smithi105 ceph-mon[24602]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T01:22:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:52 smithi105 ceph-mon[24602]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T01:22:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:53.397 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38157]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:53.397 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38157]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:53.397 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38157]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:53.651 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38182]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:53.651 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38182]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:53.651 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38182]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:53.912 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38232]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:53.912 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38232]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:53.912 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38232]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:54.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38289]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:54.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38289]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:54.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38289]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:54.171 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38272]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:54.171 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38272]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:54.171 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:22:53 smithi081 sudo[38272]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:54.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[28146]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T01:22:54.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:54.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:54.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:54.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38293]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38293]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38293]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[23696]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:54.422 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:54 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:54 smithi105 ceph-mon[24602]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T01:22:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:22:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:22:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:22:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:54 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:54.928 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38316]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:54.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38316]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:54.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38316]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:55.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38323]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:55.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38323]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:55.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:54 smithi081 sudo[38323]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26091]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26091]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26091]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26094]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26094]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T01:22:55.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 sudo[26094]: pam_unix(sudo:session): session closed for user root 2024-09-17T01:22:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi081:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:55.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:22:55.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:22:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1422890591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]': finished 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3267512506' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1422890591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]': finished 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:56.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:56 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3267512506' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:56.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1422890591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]: dispatch 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7eab9ad8-94d4-43d0-a673-101ef5615e21"}]': finished 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:56.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:56 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3267512506' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:22:59.355 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:22:59 smithi081 ceph-mon[23696]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:59.356 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:22:59 smithi081 ceph-mon[28146]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:22:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:22:59 smithi105 ceph-mon[24602]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:23:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:01 smithi105 ceph-mon[24602]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T01:23:01.417 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:01 smithi081 ceph-mon[23696]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T01:23:01.417 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:01 smithi081 ceph-mon[28146]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T01:23:02.320 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:23:02.321 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:02.321 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:23:02.321 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:23:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:03.155 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:03 smithi081 ceph-mon[23696]: Deploying daemon osd.3 on smithi081 2024-09-17T01:23:03.156 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:03 smithi081 ceph-mon[23696]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T01:23:03.156 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:03 smithi081 ceph-mon[28146]: Deploying daemon osd.3 on smithi081 2024-09-17T01:23:03.156 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:03 smithi081 ceph-mon[28146]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T01:23:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:03 smithi105 ceph-mon[24602]: Deploying daemon osd.3 on smithi081 2024-09-17T01:23:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:03 smithi105 ceph-mon[24602]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T01:23:05.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:05 smithi081 ceph-mon[23696]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T01:23:05.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:05 smithi081 ceph-mon[28146]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T01:23:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:05 smithi105 ceph-mon[24602]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T01:23:07.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[28146]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-09-17T01:23:07.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:07.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:07.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:07.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:07.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[23696]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-09-17T01:23:07.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:07.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:07.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:07.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:07 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:07 smithi105 ceph-mon[24602]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-09-17T01:23:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:07 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:07 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:07 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:07 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:09 smithi081 ceph-mon[23696]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:09.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:09 smithi081 ceph-mon[28146]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:09 smithi105 ceph-mon[24602]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:09.634 INFO:teuthology.orchestra.run.smithi081.stdout:Created osd(s) 3 on host 'smithi081' 2024-09-17T01:23:10.238 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:10 smithi081 ceph-mon[23696]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:23:10.239 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:10 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:10.239 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:10 smithi081 ceph-mon[28146]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:23:10.239 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:10 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:10.261 DEBUG:teuthology.orchestra.run.smithi081:osd.3> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T01:23:10.264 INFO:tasks.cephadm:Deploying osd.4 on smithi105 with /dev/vg_nvme/lv_4... 2024-09-17T01:23:10.264 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T01:23:10.291 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:10 smithi105 ceph-mon[24602]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:23:10.292 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:10 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:11.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:11.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:23:11.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T01:23:11.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:11.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:23:11.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:11 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:11.351 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T01:23:11.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:23:11.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T01:23:11.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:11.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:23:11.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:11.352 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[39796]: 2024-09-17T01:23:11.044+0000 7f7eda467700 -1 osd.3 0 waiting for initial osdmap 2024-09-17T01:23:11.352 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:23:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[39796]: 2024-09-17T01:23:11.048+0000 7f7ed3dfc700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:23:12.296 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[23696]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:23:12.296 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[23696]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T01:23:12.296 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.296 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[28146]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:23:12.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[28146]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T01:23:12.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:12 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:12 smithi105 ceph-mon[24602]: from='osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]': finished 2024-09-17T01:23:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:12 smithi105 ceph-mon[24602]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T01:23:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:12 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:12 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:12.905 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:23:12.924 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_4 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892] boot 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T01:23:13.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:13 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892] boot 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T01:23:13.335 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: osd.3 [v2:172.21.15.81:6826/1912267892,v1:172.21.15.81:6827/1912267892] boot 2024-09-17T01:23:13.336 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T01:23:13.336 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:13 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:23:14.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T01:23:14.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:14.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:14 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T01:23:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:14.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:14 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi081 2024-09-17T01:23:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi081 to 4016M 2024-09-17T01:23:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:15 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi081 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi081 to 4016M 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi081 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi081 to 4016M 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:15.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:15.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:15 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:16.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:16 smithi105 ceph-mon[24602]: from='client.34199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:16 smithi081 ceph-mon[23696]: from='client.34199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:16.557 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:16 smithi081 ceph-mon[28146]: from='client.34199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/4079357153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]': finished 2024-09-17T01:23:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T01:23:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:17 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/4079357153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]': finished 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T01:23:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/4079357153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]: dispatch 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1a55481a-effb-4280-8dfc-66361998f673"}]': finished 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T01:23:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:17 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:18.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:18 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/871747169' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:18 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/871747169' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:18 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/871747169' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:19.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:19 smithi105 ceph-mon[24602]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:19.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:19 smithi081 ceph-mon[23696]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:19 smithi081 ceph-mon[28146]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:21 smithi105 ceph-mon[24602]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:21 smithi081 ceph-mon[23696]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:21.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:21 smithi081 ceph-mon[28146]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:23.368 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:23 smithi105 ceph-mon[24602]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:23.369 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:23 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:23:23.369 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:23 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[28146]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:23:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:23.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[23696]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:23.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:23:23.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:23 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:24.372 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:24 smithi105 ceph-mon[24602]: Deploying daemon osd.4 on smithi105 2024-09-17T01:23:24.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:24 smithi081 ceph-mon[28146]: Deploying daemon osd.4 on smithi105 2024-09-17T01:23:24.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:24 smithi081 ceph-mon[23696]: Deploying daemon osd.4 on smithi105 2024-09-17T01:23:25.372 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:25 smithi105 ceph-mon[24602]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:25.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:25 smithi081 ceph-mon[23696]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:25.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:25 smithi081 ceph-mon[28146]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:27.340 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:27 smithi105 ceph-mon[24602]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:27 smithi081 ceph-mon[23696]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:27 smithi081 ceph-mon[28146]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:29.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:28 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:29.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:28 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:29.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:28 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:29.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:28 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:29.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:28 smithi105 ceph-mon[24602]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[28146]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:28 smithi081 ceph-mon[23696]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:30.943 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 4 on host 'smithi105' 2024-09-17T01:23:31.236 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:31 smithi105 ceph-mon[24602]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:31.236 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:31 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:31 smithi081 ceph-mon[23696]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:31 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:31 smithi081 ceph-mon[28146]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:31 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:31.543 DEBUG:teuthology.orchestra.run.smithi105:osd.4> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T01:23:31.547 INFO:tasks.cephadm:Deploying osd.5 on smithi105 with /dev/vg_nvme/lv_3... 2024-09-17T01:23:31.547 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T01:23:32.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:32 smithi105 ceph-mon[24602]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:32 smithi105 ceph-mon[24602]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:32 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[23696]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[23696]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[28146]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[28146]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:23:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:32 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:33.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:33.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:23:33.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T01:23:33.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:33.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:33.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-mon[24602]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:33.136 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[27934]: 2024-09-17T01:23:33.033+0000 7f749a98d700 -1 osd.4 0 waiting for initial osdmap 2024-09-17T01:23:33.136 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:23:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[27934]: 2024-09-17T01:23:33.037+0000 7f7493b21700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:23:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:23:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T01:23:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[23696]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: from='osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:33 smithi081 ceph-mon[28146]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:34 smithi105 ceph-mon[24602]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:34 smithi105 ceph-mon[24602]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T01:23:34.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[28146]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:34.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[28146]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T01:23:34.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[23696]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:34.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[23696]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T01:23:34.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:34.517 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:23:34.536 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_3 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: pgmap v104: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587] boot 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:35 smithi105 ceph-mon[24602]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T01:23:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:23:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:23:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: pgmap v104: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587] boot 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[23696]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: pgmap v104: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: osd.4 [v2:172.21.15.105:6800/3496403587,v1:172.21.15.105:6801/3496403587] boot 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:23:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:35 smithi081 ceph-mon[28146]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T01:23:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:36 smithi105 ceph-mon[24602]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:36 smithi105 ceph-mon[24602]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T01:23:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:36 smithi081 ceph-mon[23696]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:36 smithi081 ceph-mon[23696]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T01:23:37.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:36 smithi081 ceph-mon[28146]: pgmap v107: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:37.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:36 smithi081 ceph-mon[28146]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.068 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:38.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:38.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:37 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T01:23:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:37 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:38 smithi105 ceph-mon[24602]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:38 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi105 2024-09-17T01:23:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:38 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-17T01:23:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:38 smithi105 ceph-mon[24602]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:38 smithi105 ceph-mon[24602]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T01:23:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[28146]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi105 2024-09-17T01:23:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-17T01:23:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[28146]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[28146]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T01:23:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[23696]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:23:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi105 2024-09-17T01:23:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-17T01:23:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[23696]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:38 smithi081 ceph-mon[23696]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T01:23:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:39 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/3728929529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:39 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:39 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]': finished 2024-09-17T01:23:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:39 smithi105 ceph-mon[24602]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T01:23:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/3728929529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]': finished 2024-09-17T01:23:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[23696]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T01:23:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/3728929529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]: dispatch 2024-09-17T01:23:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cef94345-f1c8-4c87-b6d8-afdf206ced2d"}]': finished 2024-09-17T01:23:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[28146]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T01:23:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:40 smithi105 ceph-mon[24602]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T01:23:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:40 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/2949352359' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:41.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:40 smithi081 ceph-mon[23696]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T01:23:41.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:40 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/2949352359' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:41.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:40 smithi081 ceph-mon[28146]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T01:23:41.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:40 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/2949352359' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:23:43.106 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:42 smithi105 ceph-mon[24602]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:23:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:42 smithi081 ceph-mon[23696]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:23:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:42 smithi081 ceph-mon[28146]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:23:45.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:44 smithi105 ceph-mon[24602]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T01:23:45.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:44 smithi081 ceph-mon[23696]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T01:23:45.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:45 smithi081 ceph-mon[28146]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T01:23:46.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:23:46.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:46.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:23:46.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:23:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:23:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:23:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:23:46.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:23:46.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:46.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:23:46.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:23:47.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:47 smithi105 ceph-mon[24602]: Deploying daemon osd.5 on smithi105 2024-09-17T01:23:47.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:47 smithi105 ceph-mon[24602]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T01:23:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:47 smithi081 ceph-mon[23696]: Deploying daemon osd.5 on smithi105 2024-09-17T01:23:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:47 smithi081 ceph-mon[23696]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T01:23:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:47 smithi081 ceph-mon[28146]: Deploying daemon osd.5 on smithi105 2024-09-17T01:23:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:47 smithi081 ceph-mon[28146]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T01:23:49.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:49 smithi105 ceph-mon[24602]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:23:49.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:49 smithi081 ceph-mon[23696]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:23:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:49 smithi081 ceph-mon[28146]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:23:51.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:51 smithi105 ceph-mon[24602]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:51.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:51.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:51.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:51.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:51 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[23696]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[28146]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:51.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:51 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:53.335 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:53 smithi105 ceph-mon[24602]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:53 smithi081 ceph-mon[23696]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:53.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:53 smithi081 ceph-mon[28146]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:54.239 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 5 on host 'smithi105' 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:54.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:54.596 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:54 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:54.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:23:54.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:23:54.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:54 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:23:54.945 DEBUG:teuthology.orchestra.run.smithi105:osd.5> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T01:23:54.947 INFO:tasks.cephadm:Deploying osd.6 on smithi105 with /dev/vg_nvme/lv_2... 2024-09-17T01:23:54.948 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T01:23:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-mon[24602]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:23:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-mon[24602]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T01:23:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-mon[24602]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-mon[24602]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:55.636 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[30862]: 2024-09-17T01:23:55.377+0000 7fb200e37700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T01:23:55.636 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:23:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[30862]: 2024-09-17T01:23:55.382+0000 7fb1f97ca700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:23:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[23696]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:23:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[23696]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T01:23:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[23696]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[23696]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[28146]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:23:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[28146]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T01:23:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[28146]: from='osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:55 smithi081 ceph-mon[28146]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:23:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T01:23:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162] boot 2024-09-17T01:23:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T01:23:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:56 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162] boot 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T01:23:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: osd.5 [v2:172.21.15.105:6808/47016162,v1:172.21.15.105:6809/47016162] boot 2024-09-17T01:23:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T01:23:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:56 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:23:57.470 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:57 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:23:57.471 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:57 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:23:57.471 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:57 smithi105 ceph-mon[24602]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T01:23:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:23:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:23:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[23696]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T01:23:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:23:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:23:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:57 smithi081 ceph-mon[28146]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T01:23:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:58 smithi105 ceph-mon[24602]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:23:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:58 smithi105 ceph-mon[24602]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T01:23:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:58 smithi081 ceph-mon[23696]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:23:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:58 smithi081 ceph-mon[23696]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T01:23:58.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:58 smithi081 ceph-mon[28146]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:23:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:58 smithi081 ceph-mon[28146]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T01:23:59.591 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi105 2024-09-17T01:23:59.591 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.591 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.591 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.591 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-17T01:23:59.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:23:59 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.879 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:23:59.898 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_2 2024-09-17T01:23:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi105 2024-09-17T01:23:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi105 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:23:59.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:23:59 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:00.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:00 smithi105 ceph-mon[24602]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T01:24:00.863 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:00 smithi105 ceph-mon[24602]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:00 smithi081 ceph-mon[23696]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T01:24:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:00 smithi081 ceph-mon[23696]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:00.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:00 smithi081 ceph-mon[28146]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T01:24:00.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:00 smithi081 ceph-mon[28146]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:01.795 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:01 smithi105 ceph-mon[24602]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T01:24:01.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:01 smithi081 ceph-mon[23696]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T01:24:01.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:01 smithi081 ceph-mon[28146]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T01:24:02.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:02 smithi105 ceph-mon[24602]: from='client.34253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:02.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:02.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:02.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:02 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:02.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:02 smithi105 ceph-mon[24602]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[23696]: from='client.34253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[23696]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:02.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[28146]: from='client.34253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:02.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:02.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:02.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:02.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:02 smithi081 ceph-mon[28146]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/4176068690' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]': finished 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:05 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/3615248094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/4176068690' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]': finished 2024-09-17T01:24:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/3615248094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/4176068690' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]: dispatch 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b7deb9f-723d-4952-9bc5-72bf76c1aec8"}]': finished 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:05 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/3615248094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:07 smithi105 ceph-mon[24602]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:07 smithi081 ceph-mon[23696]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:07 smithi081 ceph-mon[28146]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:09.256 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:09 smithi105 ceph-mon[24602]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:09 smithi081 ceph-mon[23696]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:09.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:09 smithi081 ceph-mon[28146]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:11.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:11 smithi105 ceph-mon[24602]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:11.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:11 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:24:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:11 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[23696]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:24:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[28146]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:24:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:11 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:12.070 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:12 smithi105 ceph-mon[24602]: Deploying daemon osd.6 on smithi105 2024-09-17T01:24:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:12 smithi081 ceph-mon[23696]: Deploying daemon osd.6 on smithi105 2024-09-17T01:24:12.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:12 smithi081 ceph-mon[28146]: Deploying daemon osd.6 on smithi105 2024-09-17T01:24:13.357 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:13 smithi105 ceph-mon[24602]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:13 smithi081 ceph-mon[23696]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:13.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:13 smithi081 ceph-mon[28146]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:15 smithi105 ceph-mon[24602]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:15 smithi081 ceph-mon[23696]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:15 smithi081 ceph-mon[28146]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:16.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:16 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:16.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:16 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:16.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:16 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:16.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:16 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:16.675 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:16 smithi105 ceph-mon[24602]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:16.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:16.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:16.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:16.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[28146]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:16 smithi081 ceph-mon[23696]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:18.359 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 6 on host 'smithi105' 2024-09-17T01:24:19.018 DEBUG:teuthology.orchestra.run.smithi105:osd.6> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T01:24:19.020 INFO:tasks.cephadm:Deploying osd.7 on smithi105 with /dev/vg_nvme/lv_1... 2024-09-17T01:24:19.021 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T01:24:19.260 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:19 smithi105 ceph-mon[24602]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:19.260 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:19 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:19.260 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:19 smithi105 ceph-mon[24602]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:19.260 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:19 smithi105 ceph-mon[24602]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[28146]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[28146]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[28146]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:19.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[23696]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:19.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:19.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[23696]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:19.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:19 smithi081 ceph-mon[23696]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:24:20.328 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:20.328 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:24:20.329 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T01:24:20.329 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:20.329 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.329 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.329 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-mon[24602]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:20.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[33740]: 2024-09-17T01:24:20.372+0000 7fd300969700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T01:24:20.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:24:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[33740]: 2024-09-17T01:24:20.379+0000 7fd2fb300700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:24:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[28146]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: from='osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:20.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:20 smithi081 ceph-mon[23696]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T01:24:21.514 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:21 smithi105 ceph-mon[24602]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:21.514 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:21 smithi105 ceph-mon[24602]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T01:24:21.514 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:21 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:21.514 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:21 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[23696]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[23696]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T01:24:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:21.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[28146]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:21.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[28146]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T01:24:21.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:21.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:21 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:22.466 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T01:24:22.485 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_1 2024-09-17T01:24:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:24:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:24:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564] boot 2024-09-17T01:24:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T01:24:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:22.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:22.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:22 smithi105 ceph-mon[24602]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T01:24:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:24:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:24:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564] boot 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[23696]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: osd.6 [v2:172.21.15.105:6816/1154707564,v1:172.21.15.105:6817/1154707564] boot 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:24:22.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:22.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:22 smithi081 ceph-mon[28146]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T01:24:24.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:24 smithi105 ceph-mon[24602]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T01:24:24.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:24 smithi105 ceph-mon[24602]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:24 smithi081 ceph-mon[23696]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T01:24:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:24 smithi081 ceph-mon[23696]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:24 smithi081 ceph-mon[28146]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T01:24:24.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:24 smithi081 ceph-mon[28146]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:25.624 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:25 smithi105 ceph-mon[24602]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T01:24:25.624 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:25 smithi105 ceph-mon[24602]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:25.624 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:25 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:25.624 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:25 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:25.624 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:25 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[23696]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T01:24:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[23696]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[28146]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[28146]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T01:24:25.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:25 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T01:24:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi105 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:26 smithi105 ceph-mon[24602]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T01:24:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi105 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[23696]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi105 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:26.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:26.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:26.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:26 smithi081 ceph-mon[28146]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:27 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/3416721568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:27 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:27 smithi105 ceph-mon[24602]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]': finished 2024-09-17T01:24:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:27 smithi105 ceph-mon[24602]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T01:24:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:27 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:27.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/3416721568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[23696]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]': finished 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[23696]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/3416721568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]: dispatch 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[28146]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ff4827-98dc-43d1-91df-247316cce10e"}]': finished 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[28146]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T01:24:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:27 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:28.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:28 smithi105 ceph-mon[24602]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-17T01:24:28.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:28 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/1238924248' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:28.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:28 smithi081 ceph-mon[23696]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-17T01:24:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:28 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/1238924248' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:28 smithi081 ceph-mon[28146]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-17T01:24:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:28 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/1238924248' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T01:24:31.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:31 smithi105 ceph-mon[24602]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:31 smithi081 ceph-mon[23696]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:31 smithi081 ceph-mon[28146]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:33.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:33 smithi105 ceph-mon[24602]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-17T01:24:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:33 smithi081 ceph-mon[23696]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-17T01:24:33.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:33 smithi081 ceph-mon[28146]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-17T01:24:34.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:24:34.303 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:34 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:34.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:24:34.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:34 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:34.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:24:34.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:34 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:35.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:35 smithi105 ceph-mon[24602]: Deploying daemon osd.7 on smithi105 2024-09-17T01:24:35.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:35 smithi105 ceph-mon[24602]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-17T01:24:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:35 smithi081 ceph-mon[23696]: Deploying daemon osd.7 on smithi105 2024-09-17T01:24:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:35 smithi081 ceph-mon[23696]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-17T01:24:35.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:35 smithi081 ceph-mon[28146]: Deploying daemon osd.7 on smithi105 2024-09-17T01:24:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:35 smithi081 ceph-mon[28146]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-17T01:24:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:37 smithi105 ceph-mon[24602]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:24:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:37 smithi081 ceph-mon[23696]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:24:37.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:37 smithi081 ceph-mon[28146]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:24:39.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:39 smithi105 ceph-mon[24602]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:39.353 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:39.353 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:39.353 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:39.353 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:39 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[23696]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[28146]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:39.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:39.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:39.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:39 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:41 smithi105 ceph-mon[24602]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:41.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:41 smithi081 ceph-mon[23696]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:41.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:41 smithi081 ceph-mon[28146]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:42.282 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 7 on host 'smithi105' 2024-09-17T01:24:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:42 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:42 smithi105 ceph-mon[24602]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:42 smithi105 ceph-mon[24602]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:42 smithi105 ceph-mon[24602]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:42.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:42 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:42.914 DEBUG:teuthology.orchestra.run.smithi105:osd.7> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T01:24:42.916 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T01:24:42.916 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T01:24:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[23696]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[23696]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[23696]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[28146]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[28146]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:24:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[28146]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:42 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-mon[24602]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:24:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-mon[24602]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T01:24:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-mon[24602]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-mon[24602]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:43.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[36645]: 2024-09-17T01:24:43.518+0000 7f5ee0ddb700 -1 osd.7 0 waiting for initial osdmap 2024-09-17T01:24:43.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:24:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[36645]: 2024-09-17T01:24:43.525+0000 7f5ed976e700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[23696]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[23696]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[23696]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[23696]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[28146]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:24:43.787 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[28146]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T01:24:43.788 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:43.788 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[28146]: from='osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:43.788 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:43 smithi081 ceph-mon[28146]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:24:44.571 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:44.571 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T01:24:44.571 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.571 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:44.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934] boot 2024-09-17T01:24:44.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T01:24:44.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:44 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T01:24:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934] boot 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:24:44.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: osd.7 [v2:172.21.15.105:6824/3224354934,v1:172.21.15.105:6825/3224354934] boot 2024-09-17T01:24:44.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T01:24:44.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:44 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:24:44.789 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:24:45.459 INFO:teuthology.orchestra.run.smithi081.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726536284,"num_in_osds":8,"osd_in_since":1726536267,"num_remapped_pgs":1} 2024-09-17T01:24:45.460 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:24:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:45 smithi105 ceph-mon[24602]: purged_snaps scrub starts 2024-09-17T01:24:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:45 smithi105 ceph-mon[24602]: purged_snaps scrub ok 2024-09-17T01:24:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:45 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2640079168' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T01:24:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:45 smithi105 ceph-mon[24602]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[23696]: purged_snaps scrub starts 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[23696]: purged_snaps scrub ok 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2640079168' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[23696]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[28146]: purged_snaps scrub starts 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[28146]: purged_snaps scrub ok 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2640079168' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T01:24:45.700 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:45 smithi081 ceph-mon[28146]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T01:24:45.713 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:24:46.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:24:46.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: pgmap v166: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: Detected new or changed devices on smithi105 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:46.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:46.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:46.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:46 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:24:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:24:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: pgmap v166: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi105 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: pgmap v166: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi105 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:24:46.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:46.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:46.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:46.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:46.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:46 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:47.609 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:24:47.609 INFO:teuthology.orchestra.run.smithi081.stdout:{"epoch":58,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","created":"2024-09-17T01:19:46.321237+0000","modified":"2024-09-17T01:24:46.984479+0000","last_up_change":"2024-09-17T01:24:44.505394+0000","last_in_change":"2024-09-17T01:24:27.487620+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T01:22:50.230358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfcee6db-e2f0-4f9f-bae8-a55f4671b506","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6802","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6803","nonce":2140831668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6804","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6805","nonce":2140831668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6808","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6809","nonce":2140831668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6806","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6807","nonce":2140831668}]},"public_addr":"172.21.15.81:6803/2140831668","cluster_addr":"172.21.15.81:6805/2140831668","heartbeat_back_addr":"172.21.15.81:6809/2140831668","heartbeat_front_addr":"172.21.15.81:6807/2140831668","state":["exists","up"]},{"osd":1,"uuid":"59a4263a-67b4-4f17-8757-22aba74776a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6810","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6811","nonce":2547106968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6812","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6813","nonce":2547106968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6816","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6817","nonce":2547106968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6814","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6815","nonce":2547106968}]},"public_addr":"172.21.15.81:6811/2547106968","cluster_addr":"172.21.15.81:6813/2547106968","heartbeat_back_addr":"172.21.15.81:6817/2547106968","heartbeat_front_addr":"172.21.15.81:6815/2547106968","state":["exists","up"]},{"osd":2,"uuid":"31cb2977-5e2e-462e-98b9-064f54a364c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6818","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6819","nonce":2272030537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6820","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6821","nonce":2272030537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6824","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6825","nonce":2272030537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6822","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6823","nonce":2272030537}]},"public_addr":"172.21.15.81:6819/2272030537","cluster_addr":"172.21.15.81:6821/2272030537","heartbeat_back_addr":"172.21.15.81:6825/2272030537","heartbeat_front_addr":"172.21.15.81:6823/2272030537","state":["exists","up"]},{"osd":3,"uuid":"7eab9ad8-94d4-43d0-a673-101ef5615e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6826","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6827","nonce":1912267892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6828","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6829","nonce":1912267892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6832","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6833","nonce":1912267892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6830","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6831","nonce":1912267892}]},"public_addr":"172.21.15.81:6827/1912267892","cluster_addr":"172.21.15.81:6829/1912267892","heartbeat_back_addr":"172.21.15.81:6833/1912267892","heartbeat_front_addr":"172.21.15.81:6831/1912267892","state":["exists","up"]},{"osd":4,"uuid":"1a55481a-effb-4280-8dfc-66361998f673","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6801","nonce":3496403587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6803","nonce":3496403587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6807","nonce":3496403587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6805","nonce":3496403587}]},"public_addr":"172.21.15.105:6801/3496403587","cluster_addr":"172.21.15.105:6803/3496403587","heartbeat_back_addr":"172.21.15.105:6807/3496403587","heartbeat_front_addr":"172.21.15.105:6805/3496403587","state":["exists","up"]},{"osd":5,"uuid":"cef94345-f1c8-4c87-b6d8-afdf206ced2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6809","nonce":47016162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6811","nonce":47016162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6815","nonce":47016162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6813","nonce":47016162}]},"public_addr":"172.21.15.105:6809/47016162","cluster_addr":"172.21.15.105:6811/47016162","heartbeat_back_addr":"172.21.15.105:6815/47016162","heartbeat_front_addr":"172.21.15.105:6813/47016162","state":["exists","up"]},{"osd":6,"uuid":"2b7deb9f-723d-4952-9bc5-72bf76c1aec8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6817","nonce":1154707564}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6819","nonce":1154707564}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6823","nonce":1154707564}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6821","nonce":1154707564}]},"public_addr":"172.21.15.105:6817/1154707564","cluster_addr":"172.21.15.105:6819/1154707564","heartbeat_back_addr":"172.21.15.105:6823/1154707564","heartbeat_front_addr":"172.21.15.105:6821/1154707564","state":["exists","up"]},{"osd":7,"uuid":"a3ff4827-98dc-43d1-91df-247316cce10e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6825","nonce":3224354934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6827","nonce":3224354934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6831","nonce":3224354934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6829","nonce":3224354934}]},"public_addr":"172.21.15.105:6825/3224354934","cluster_addr":"172.21.15.105:6827/3224354934","heartbeat_back_addr":"172.21.15.105:6831/3224354934","heartbeat_front_addr":"172.21.15.105:6829/3224354934","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:02.089067+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:25.314677+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:47.369379+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:10.554919+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:32.078351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:54.876068+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:19.508161+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:42.843973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.81:6801/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:6800/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/742413165":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/662934856":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/2620727570":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/3873552392":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1710452097":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6800/597110760":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1436016564":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6801/597110760":"2024-09-18T01:20:24.007831+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:24:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-17T01:24:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[23696]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T01:24:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1760093927' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:24:48.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-17T01:24:48.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[28146]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T01:24:48.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1760093927' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:24:48.225 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-17T01:22:50.230358+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-17T01:24:48.225 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-17T01:24:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:47 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-17T01:24:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:47 smithi105 ceph-mon[24602]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T01:24:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1760093927' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:24:48.428 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:24:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:48 smithi081 ceph-mon[23696]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:48 smithi081 ceph-mon[23696]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T01:24:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:48 smithi081 ceph-mon[28146]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:48 smithi081 ceph-mon[28146]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T01:24:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:48 smithi105 ceph-mon[24602]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:24:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:48 smithi105 ceph-mon[24602]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T01:24:49.978 INFO:teuthology.orchestra.run.smithi081.stdout:pg_num: 1 2024-09-17T01:24:50.516 INFO:tasks.cephadm:Adding prometheus.a on smithi105 2024-09-17T01:24:50.516 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi105=a' 2024-09-17T01:24:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:50 smithi105 ceph-mon[24602]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-17T01:24:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:50 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1443731344' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T01:24:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:50 smithi081 ceph-mon[23696]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-17T01:24:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:50 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1443731344' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T01:24:51.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:50 smithi081 ceph-mon[28146]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-17T01:24:51.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:50 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1443731344' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T01:24:52.515 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled prometheus update... 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:52 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T01:24:53.182 DEBUG:teuthology.orchestra.run.smithi105:prometheus.a> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-17T01:24:53.184 INFO:tasks.cephadm:Adding node-exporter.a on smithi081 2024-09-17T01:24:53.184 INFO:tasks.cephadm:Adding node-exporter.b on smithi105 2024-09-17T01:24:53.184 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi081=a;smithi105=b' 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' 2024-09-17T01:24:53.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:52 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T01:24:53.772 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ignoring --setuser ceph since I am not root 2024-09-17T01:24:53.818 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ignoring --setgroup ceph since I am not root 2024-09-17T01:24:53.818 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-mgr[25775]: -- 172.21.15.105:0/413251614 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5559e8828340 con 0x5559e8801000 2024-09-17T01:24:53.819 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:53.729+0000 7f56cdf3e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:24:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setuser ceph since I am not root 2024-09-17T01:24:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setgroup ceph since I am not root 2024-09-17T01:24:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mgr[23930]: -- 172.21.15.81:0/571661737 <== mon.1 v2:172.21.15.81:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563e61b94340 con 0x563e61b6c800 2024-09-17T01:24:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:53.736+0000 7fb63b283000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:24:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:53.813+0000 7fb63b283000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:24:54.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-mon[24602]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:54.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-mon[24602]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T01:24:54.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-mon[24602]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T01:24:54.069 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-mon[24602]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-17T01:24:54.070 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:53.810+0000 7f56cdf3e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:24:54.070 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:53.984+0000 7f56cdf3e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:24:54.070 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:54.071+0000 7f56cdf3e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:24:54.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:54.203+0000 7f56cdf3e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:24:54.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[23696]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[23696]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[23696]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[23696]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:53.986+0000 7fb63b283000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.074+0000 7fb63b283000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.203+0000 7fb63b283000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:24:54.409 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.410+0000 7fb63b283000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:24:54.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[28146]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:24:54.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[28146]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T01:24:54.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[28146]: from='mgr.14152 172.21.15.81:0/3293814126' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T01:24:54.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:53 smithi081 ceph-mon[28146]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-17T01:24:54.732 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:54.414+0000 7f56cdf3e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:24:54.982 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.732+0000 7fb63b283000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:24:54.982 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.819+0000 7fb63b283000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:24:54.982 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:54.984+0000 7fb63b283000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:24:55.015 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:54.753+0000 7f56cdf3e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:24:55.015 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:54.842+0000 7f56cdf3e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:24:55.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:55.016+0000 7f56cdf3e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:24:56.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:56.062+0000 7f56cdf3e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:24:56.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:56.152+0000 7f56cdf3e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:24:56.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:56 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:56.022+0000 7fb63b283000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:24:56.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:56 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:56.113+0000 7fb63b283000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:24:57.316 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.008+0000 7fb63b283000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:24:57.344 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.040+0000 7f56cdf3e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.317+0000 7fb63b283000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.450+0000 7fb63b283000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.538+0000 7fb63b283000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.636+0000 7fb63b283000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.346+0000 7f56cdf3e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.479+0000 7f56cdf3e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:24:57.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.567+0000 7f56cdf3e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:24:57.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.725+0000 7fb63b283000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:24:57.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.873+0000 7fb63b283000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:24:57.990 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.665+0000 7f56cdf3e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:24:57.991 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.752+0000 7f56cdf3e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:24:57.991 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.900+0000 7f56cdf3e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:24:57.991 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:57.992+0000 7f56cdf3e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:24:58.346 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:57.967+0000 7fb63b283000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:24:58.346 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:58.063+0000 7fb63b283000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:24:58.371 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:58.090+0000 7f56cdf3e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:24:58.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:58.373+0000 7f56cdf3e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:24:58.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:58.461+0000 7f56cdf3e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:24:58.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:58.549+0000 7f56cdf3e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:24:58.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:58.348+0000 7fb63b283000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:24:58.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:58.437+0000 7fb63b283000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:24:58.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:58.527+0000 7fb63b283000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:24:59.861 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: Active manager daemon y restarted 2024-09-17T01:24:59.861 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: Activating manager daemon y 2024-09-17T01:24:59.861 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: Standby manager daemon x restarted 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:24:59.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:24:59.863 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:24:59.600+0000 7fb63b283000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:24:59.863 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:24:59] ENGINE Bus STARTING 2024-09-17T01:24:59.863 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: Active manager daemon y restarted 2024-09-17T01:24:59.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: Activating manager daemon y 2024-09-17T01:24:59.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T01:24:59.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: Standby manager daemon x restarted 2024-09-17T01:24:59.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:24:59.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:24:59.865 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:24:59.865 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:24:59.865 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:24:59.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: Active manager daemon y restarted 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: Activating manager daemon y 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: Standby manager daemon x restarted 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:24:59.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/929595991' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:59.613+0000 7f56cdf3e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: [17/Sep/2024:01:24:59] ENGINE Bus STARTING 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: CherryPy Checker: 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: The Application mounted at '' has an empty config. 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: [17/Sep/2024:01:24:59] ENGINE Serving on http://:::9283 2024-09-17T01:24:59.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:24:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: [17/Sep/2024:01:24:59] ENGINE Bus STARTED 2024-09-17T01:25:00.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: CherryPy Checker: 2024-09-17T01:25:00.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: The Application mounted at '' has an empty config. 2024-09-17T01:25:00.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:25:00.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:24:59] ENGINE Serving on http://:::9283 2024-09-17T01:25:00.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:24:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:24:59] ENGINE Bus STARTED 2024-09-17T01:25:00.512 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:25:00] ENGINE Bus STARTING 2024-09-17T01:25:00.513 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:25:00] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:25:00.513 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:25:00] ENGINE Bus STARTED 2024-09-17T01:25:00.683 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled node-exporter update... 2024-09-17T01:25:00.875 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: mgrmap e17: y(active, starting, since 0.0585146s), standbys: x 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:25:00.876 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: Manager daemon y is now available 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: [17/Sep/2024:01:25:00] ENGINE Bus STARTING 2024-09-17T01:25:00.877 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: [17/Sep/2024:01:25:00] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:25:00.878 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: [17/Sep/2024:01:25:00] ENGINE Bus STARTED 2024-09-17T01:25:00.878 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: mgrmap e17: y(active, starting, since 0.0585146s), standbys: x 2024-09-17T01:25:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:25:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:25:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:25:00.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:25:00.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: [17/Sep/2024:01:25:00] ENGINE Bus STARTING 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: [17/Sep/2024:01:25:00] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: [17/Sep/2024:01:25:00] ENGINE Bus STARTED 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: mgrmap e17: y(active, starting, since 0.0585146s), standbys: x 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:25:00.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: Manager daemon y is now available 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: [17/Sep/2024:01:25:00] ENGINE Bus STARTING 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: [17/Sep/2024:01:25:00] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: [17/Sep/2024:01:25:00] ENGINE Bus STARTED 2024-09-17T01:25:00.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.530 DEBUG:teuthology.orchestra.run.smithi081:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-17T01:25:01.532 DEBUG:teuthology.orchestra.run.smithi105:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-17T01:25:01.534 INFO:tasks.cephadm:Adding alertmanager.a on smithi081 2024-09-17T01:25:01.535 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi081=a' 2024-09-17T01:25:01.829 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: mgrmap e18: y(active, since 1.07414s), standbys: x 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi081=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: Saving service node-exporter spec with placement smithi081=a;smithi105=b;count:2 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: mgrmap e18: y(active, since 1.07414s), standbys: x 2024-09-17T01:25:01.830 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi081=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: Saving service node-exporter spec with placement smithi081=a;smithi105=b;count:2 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.831 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:01 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: mgrmap e18: y(active, since 1.07414s), standbys: x 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi081=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: Saving service node-exporter spec with placement smithi081=a;smithi105=b;count:2 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:01.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:01 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:25:02.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:02 smithi105 ceph-mon[24602]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-17T01:25:03.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:25:03.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[23696]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-17T01:25:03.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:03.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:03.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:25:03.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:02 smithi081 ceph-mon[28146]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-17T01:25:03.679 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled alertmanager update... 2024-09-17T01:25:03.987 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:03 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:03.987 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:03 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.987 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:03 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:03.988 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:03 smithi105 ceph-mon[24602]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:25:03.988 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:03 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[23696]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[28146]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:25:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:03 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:04.463 DEBUG:teuthology.orchestra.run.smithi081:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-17T01:25:04.466 INFO:tasks.cephadm:Adding grafana.a on smithi105 2024-09-17T01:25:04.466 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi105=a' 2024-09-17T01:25:05.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:04 smithi105 ceph-mon[24602]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:05.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:04 smithi105 ceph-mon[24602]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi081=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:05.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:04 smithi105 ceph-mon[24602]: Saving service alertmanager spec with placement smithi081=a;count:1 2024-09-17T01:25:05.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[28146]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:05.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[28146]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi081=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:05.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[28146]: Saving service alertmanager spec with placement smithi081=a;count:1 2024-09-17T01:25:05.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[23696]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:05.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[23696]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi081=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:05.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:04 smithi081 ceph-mon[23696]: Saving service alertmanager spec with placement smithi081=a;count:1 2024-09-17T01:25:07.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:06 smithi105 ceph-mon[24602]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:07.316 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled grafana update... 2024-09-17T01:25:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:06 smithi081 ceph-mon[28146]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:06 smithi081 ceph-mon[23696]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:08.156 DEBUG:teuthology.orchestra.run.smithi105:grafana.a> sudo journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service 2024-09-17T01:25:08.158 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T01:25:08.159 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T01:25:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:08 smithi105 ceph-mon[24602]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:08 smithi105 ceph-mon[24602]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T01:25:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:08 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:08.465 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[23696]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:08.465 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[23696]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T01:25:08.465 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:08.466 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[28146]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:25:08.466 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[28146]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T01:25:08.466 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:08 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:09.627 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:09 smithi081 ceph-mon[23696]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:09.627 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:09 smithi081 ceph-mon[28146]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:09 smithi105 ceph-mon[24602]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:10.122 INFO:teuthology.orchestra.run.smithi081.stdout:[client.0] 2024-09-17T01:25:10.122 INFO:teuthology.orchestra.run.smithi081.stdout: key = AQB22uhm1/4bBxAAGNufTXj3Iybj1Yec01rRUw== 2024-09-17T01:25:10.713 DEBUG:teuthology.orchestra.run.smithi081:> set -ex 2024-09-17T01:25:10.713 DEBUG:teuthology.orchestra.run.smithi081:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T01:25:10.713 DEBUG:teuthology.orchestra.run.smithi081:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T01:25:10.750 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T01:25:10.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[23696]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:10.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:10.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:10 smithi105 ceph-mon[24602]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:10 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:10 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[28146]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:10 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2867993802' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:13.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:12 smithi105 ceph-mon[24602]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:12 smithi081 ceph-mon[23696]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:13.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:12 smithi081 ceph-mon[28146]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:14.650 INFO:teuthology.orchestra.run.smithi105.stdout:[client.1] 2024-09-17T01:25:14.650 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQB62uhmvm++FxAAxCA4bDD1XXNhslNFnog9HQ== 2024-09-17T01:25:15.247 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[23696]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:15.247 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.105:0/687061806' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[28146]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.105:0/687061806' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.248 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:14 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:15.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:14 smithi105 ceph-mon[24602]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:15.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.105:0/687061806' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:14 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:25:15.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:14 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:25:15.276 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T01:25:15.277 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-17T01:25:15.277 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-17T01:25:15.319 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T01:25:15.319 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T01:25:15.319 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T01:25:15.534 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:15.811 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:25:15 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:25:17.026 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[23696]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:17.026 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:17.027 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[23696]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:25:17.027 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[28146]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:17.027 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:17.027 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:16 smithi081 ceph-mon[28146]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:25:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:16 smithi105 ceph-mon[24602]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:16 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:16 smithi105 ceph-mon[24602]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:25:17.409 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:25:18.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:17 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/206272898' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:25:18.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/206272898' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:25:18.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:17 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/206272898' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:25:18.558 INFO:teuthology.orchestra.run.smithi081.stdout:{"epoch":19,"active_gid":14412,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6800","nonce":52805828},{"type":"v1","addr":"172.21.15.81:6801","nonce":52805828}]},"active_addr":"172.21.15.81:6801/52805828","active_change":"2024-09-17T01:24:59.602433+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34328,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.81:8443/","prometheus":"http://172.21.15.81:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.81:0","nonce":1770244867}]},{"addrvec":[{"type":"v2","addr":"172.21.15.81:0","nonce":4171850412}]},{"addrvec":[{"type":"v2","addr":"172.21.15.81:0","nonce":1237785981}]},{"addrvec":[{"type":"v2","addr":"172.21.15.81:0","nonce":62530244}]}]}} 2024-09-17T01:25:18.561 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T01:25:18.561 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T01:25:18.562 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:25:19.059 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:19.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:18 smithi081 ceph-mon[23696]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:19.120 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:18 smithi081 ceph-mon[28146]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:18 smithi105 ceph-mon[24602]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:20.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 podman[42792]: 2024-09-17 01:25:20.064635347 +0000 UTC m=+2.858017105 container create 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 podman[42792]: 2024-09-17 01:25:20.225005343 +0000 UTC m=+3.018387037 container init 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 podman[42792]: 2024-09-17 01:25:20.227853704 +0000 UTC m=+3.021235397 container start 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.231Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.231Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.232Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-17T01:25:20.681 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.232Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=arp 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=edac 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-17T01:25:20.682 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-17T01:25:20.683 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=os 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-17T01:25:20.684 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=stat 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=time 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=uname 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-17T01:25:20.685 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-17T01:25:20.686 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-17T01:25:20.686 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-17T01:25:20.686 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[42962]: ts=2024-09-17T01:25:20.233Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-17T01:25:20.686 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 bash[42792]: 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 2024-09-17T01:25:20.686 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:25:20 smithi081 systemd[1]: Started Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[23696]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[23696]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[28146]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.962 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:20 smithi081 ceph-mon[28146]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:25:20.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:20 smithi105 ceph-mon[24602]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:20.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:20 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:20 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:20.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:20 smithi105 ceph-mon[24602]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:25:21.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:21 smithi105 bash[39956]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-09-17T01:25:21.580 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:25:21.581 INFO:teuthology.orchestra.run.smithi081.stdout:{"epoch":60,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","created":"2024-09-17T01:19:46.321237+0000","modified":"2024-09-17T01:24:59.602097+0000","last_up_change":"2024-09-17T01:24:44.505394+0000","last_in_change":"2024-09-17T01:24:27.487620+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T01:22:50.230358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfcee6db-e2f0-4f9f-bae8-a55f4671b506","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6802","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6803","nonce":2140831668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6804","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6805","nonce":2140831668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6808","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6809","nonce":2140831668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6806","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6807","nonce":2140831668}]},"public_addr":"172.21.15.81:6803/2140831668","cluster_addr":"172.21.15.81:6805/2140831668","heartbeat_back_addr":"172.21.15.81:6809/2140831668","heartbeat_front_addr":"172.21.15.81:6807/2140831668","state":["exists","up"]},{"osd":1,"uuid":"59a4263a-67b4-4f17-8757-22aba74776a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6810","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6811","nonce":2547106968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6812","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6813","nonce":2547106968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6816","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6817","nonce":2547106968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6814","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6815","nonce":2547106968}]},"public_addr":"172.21.15.81:6811/2547106968","cluster_addr":"172.21.15.81:6813/2547106968","heartbeat_back_addr":"172.21.15.81:6817/2547106968","heartbeat_front_addr":"172.21.15.81:6815/2547106968","state":["exists","up"]},{"osd":2,"uuid":"31cb2977-5e2e-462e-98b9-064f54a364c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6818","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6819","nonce":2272030537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6820","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6821","nonce":2272030537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6824","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6825","nonce":2272030537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6822","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6823","nonce":2272030537}]},"public_addr":"172.21.15.81:6819/2272030537","cluster_addr":"172.21.15.81:6821/2272030537","heartbeat_back_addr":"172.21.15.81:6825/2272030537","heartbeat_front_addr":"172.21.15.81:6823/2272030537","state":["exists","up"]},{"osd":3,"uuid":"7eab9ad8-94d4-43d0-a673-101ef5615e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6826","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6827","nonce":1912267892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6828","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6829","nonce":1912267892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6832","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6833","nonce":1912267892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6830","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6831","nonce":1912267892}]},"public_addr":"172.21.15.81:6827/1912267892","cluster_addr":"172.21.15.81:6829/1912267892","heartbeat_back_addr":"172.21.15.81:6833/1912267892","heartbeat_front_addr":"172.21.15.81:6831/1912267892","state":["exists","up"]},{"osd":4,"uuid":"1a55481a-effb-4280-8dfc-66361998f673","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6801","nonce":3496403587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6803","nonce":3496403587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6807","nonce":3496403587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6805","nonce":3496403587}]},"public_addr":"172.21.15.105:6801/3496403587","cluster_addr":"172.21.15.105:6803/3496403587","heartbeat_back_addr":"172.21.15.105:6807/3496403587","heartbeat_front_addr":"172.21.15.105:6805/3496403587","state":["exists","up"]},{"osd":5,"uuid":"cef94345-f1c8-4c87-b6d8-afdf206ced2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6809","nonce":47016162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6811","nonce":47016162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6815","nonce":47016162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6813","nonce":47016162}]},"public_addr":"172.21.15.105:6809/47016162","cluster_addr":"172.21.15.105:6811/47016162","heartbeat_back_addr":"172.21.15.105:6815/47016162","heartbeat_front_addr":"172.21.15.105:6813/47016162","state":["exists","up"]},{"osd":6,"uuid":"2b7deb9f-723d-4952-9bc5-72bf76c1aec8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6817","nonce":1154707564}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6819","nonce":1154707564}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6823","nonce":1154707564}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6821","nonce":1154707564}]},"public_addr":"172.21.15.105:6817/1154707564","cluster_addr":"172.21.15.105:6819/1154707564","heartbeat_back_addr":"172.21.15.105:6823/1154707564","heartbeat_front_addr":"172.21.15.105:6821/1154707564","state":["exists","up"]},{"osd":7,"uuid":"a3ff4827-98dc-43d1-91df-247316cce10e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6825","nonce":3224354934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6827","nonce":3224354934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6831","nonce":3224354934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6829","nonce":3224354934}]},"public_addr":"172.21.15.105:6825/3224354934","cluster_addr":"172.21.15.105:6827/3224354934","heartbeat_back_addr":"172.21.15.105:6831/3224354934","heartbeat_front_addr":"172.21.15.105:6829/3224354934","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:02.089067+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:25.314677+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:47.369379+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:10.554919+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:32.078351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:54.876068+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:19.508161+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:42.843973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.81:0/3298429670":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/1296425100":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6801/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/1568007122":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6801/1138093002":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6800/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/742413165":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/662934856":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/2620727570":"2024-09-18T01:20:45.116020+0000","172.21.15.81:6800/1138093002":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/3873552392":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1710452097":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6800/597110760":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1807582732":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/1436016564":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6801/597110760":"2024-09-18T01:20:24.007831+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:25:21.873 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:21 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2528530178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:21.873 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:21 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2528530178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:21 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2528530178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:21.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:21 smithi105 bash[39956]: Getting image source signatures 2024-09-17T01:25:21.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:21 smithi105 bash[39956]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-09-17T01:25:21.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:21 smithi105 bash[39956]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-09-17T01:25:21.886 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:21 smithi105 bash[39956]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-09-17T01:25:22.171 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T01:25:22.172 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:25:22.377 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:22.819 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:22 smithi081 ceph-mon[23696]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:22.820 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:22 smithi081 ceph-mon[28146]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:22 smithi105 ceph-mon[24602]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:23.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:25:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:25:23.741 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:23] "GET /metrics HTTP/1.1" 200 191504 "" "Prometheus/2.33.4" 2024-09-17T01:25:23.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:23 smithi105 bash[39956]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-09-17T01:25:23.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:23 smithi105 bash[39956]: Writing manifest to image destination 2024-09-17T01:25:24.097 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:25:24.097 INFO:teuthology.orchestra.run.smithi081.stdout:{"epoch":60,"fsid":"ded6f264-7492-11ef-bceb-c7b262605968","created":"2024-09-17T01:19:46.321237+0000","modified":"2024-09-17T01:24:59.602097+0000","last_up_change":"2024-09-17T01:24:44.505394+0000","last_in_change":"2024-09-17T01:24:27.487620+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T01:22:50.230358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"bfcee6db-e2f0-4f9f-bae8-a55f4671b506","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6802","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6803","nonce":2140831668}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6804","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6805","nonce":2140831668}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6808","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6809","nonce":2140831668}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6806","nonce":2140831668},{"type":"v1","addr":"172.21.15.81:6807","nonce":2140831668}]},"public_addr":"172.21.15.81:6803/2140831668","cluster_addr":"172.21.15.81:6805/2140831668","heartbeat_back_addr":"172.21.15.81:6809/2140831668","heartbeat_front_addr":"172.21.15.81:6807/2140831668","state":["exists","up"]},{"osd":1,"uuid":"59a4263a-67b4-4f17-8757-22aba74776a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6810","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6811","nonce":2547106968}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6812","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6813","nonce":2547106968}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6816","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6817","nonce":2547106968}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6814","nonce":2547106968},{"type":"v1","addr":"172.21.15.81:6815","nonce":2547106968}]},"public_addr":"172.21.15.81:6811/2547106968","cluster_addr":"172.21.15.81:6813/2547106968","heartbeat_back_addr":"172.21.15.81:6817/2547106968","heartbeat_front_addr":"172.21.15.81:6815/2547106968","state":["exists","up"]},{"osd":2,"uuid":"31cb2977-5e2e-462e-98b9-064f54a364c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6818","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6819","nonce":2272030537}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6820","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6821","nonce":2272030537}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6824","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6825","nonce":2272030537}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6822","nonce":2272030537},{"type":"v1","addr":"172.21.15.81:6823","nonce":2272030537}]},"public_addr":"172.21.15.81:6819/2272030537","cluster_addr":"172.21.15.81:6821/2272030537","heartbeat_back_addr":"172.21.15.81:6825/2272030537","heartbeat_front_addr":"172.21.15.81:6823/2272030537","state":["exists","up"]},{"osd":3,"uuid":"7eab9ad8-94d4-43d0-a673-101ef5615e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6826","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6827","nonce":1912267892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6828","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6829","nonce":1912267892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6832","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6833","nonce":1912267892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.81:6830","nonce":1912267892},{"type":"v1","addr":"172.21.15.81:6831","nonce":1912267892}]},"public_addr":"172.21.15.81:6827/1912267892","cluster_addr":"172.21.15.81:6829/1912267892","heartbeat_back_addr":"172.21.15.81:6833/1912267892","heartbeat_front_addr":"172.21.15.81:6831/1912267892","state":["exists","up"]},{"osd":4,"uuid":"1a55481a-effb-4280-8dfc-66361998f673","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6801","nonce":3496403587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6803","nonce":3496403587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6807","nonce":3496403587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":3496403587},{"type":"v1","addr":"172.21.15.105:6805","nonce":3496403587}]},"public_addr":"172.21.15.105:6801/3496403587","cluster_addr":"172.21.15.105:6803/3496403587","heartbeat_back_addr":"172.21.15.105:6807/3496403587","heartbeat_front_addr":"172.21.15.105:6805/3496403587","state":["exists","up"]},{"osd":5,"uuid":"cef94345-f1c8-4c87-b6d8-afdf206ced2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6809","nonce":47016162}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6811","nonce":47016162}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6815","nonce":47016162}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":47016162},{"type":"v1","addr":"172.21.15.105:6813","nonce":47016162}]},"public_addr":"172.21.15.105:6809/47016162","cluster_addr":"172.21.15.105:6811/47016162","heartbeat_back_addr":"172.21.15.105:6815/47016162","heartbeat_front_addr":"172.21.15.105:6813/47016162","state":["exists","up"]},{"osd":6,"uuid":"2b7deb9f-723d-4952-9bc5-72bf76c1aec8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6817","nonce":1154707564}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6819","nonce":1154707564}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6823","nonce":1154707564}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1154707564},{"type":"v1","addr":"172.21.15.105:6821","nonce":1154707564}]},"public_addr":"172.21.15.105:6817/1154707564","cluster_addr":"172.21.15.105:6819/1154707564","heartbeat_back_addr":"172.21.15.105:6823/1154707564","heartbeat_front_addr":"172.21.15.105:6821/1154707564","state":["exists","up"]},{"osd":7,"uuid":"a3ff4827-98dc-43d1-91df-247316cce10e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6825","nonce":3224354934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6827","nonce":3224354934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6831","nonce":3224354934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3224354934},{"type":"v1","addr":"172.21.15.105:6829","nonce":3224354934}]},"public_addr":"172.21.15.105:6825/3224354934","cluster_addr":"172.21.15.105:6827/3224354934","heartbeat_back_addr":"172.21.15.105:6831/3224354934","heartbeat_front_addr":"172.21.15.105:6829/3224354934","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:02.089067+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:25.314677+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:22:47.369379+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:10.554919+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:32.078351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:23:54.876068+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:19.508161+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:24:42.843973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.81:0/3298429670":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/1296425100":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6801/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/1568007122":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6801/1138093002":"2024-09-18T01:24:59.602064+0000","172.21.15.81:6800/547777509":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/742413165":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/662934856":"2024-09-18T01:20:45.116020+0000","172.21.15.81:0/2620727570":"2024-09-18T01:20:45.116020+0000","172.21.15.81:6800/1138093002":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/3873552392":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1710452097":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6800/597110760":"2024-09-18T01:20:24.007831+0000","172.21.15.81:0/1807582732":"2024-09-18T01:24:59.602064+0000","172.21.15.81:0/1436016564":"2024-09-18T01:20:24.007831+0000","172.21.15.81:6801/597110760":"2024-09-18T01:20:24.007831+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:25:24.199 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:23 smithi105 podman[39956]: 2024-09-17 01:25:23.893228129 +0000 UTC m=+2.797946972 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-09-17T01:25:24.199 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 podman[39956]: 2024-09-17 01:25:24.00006507 +0000 UTC m=+2.904783908 container create 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:25:24.635 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 podman[39956]: 2024-09-17 01:25:24.201889012 +0000 UTC m=+3.106607847 container init 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 podman[39956]: 2024-09-17 01:25:24.205505679 +0000 UTC m=+3.110224507 container start 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.210Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.210Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.210Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=arp 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=bcache 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=bonding 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=btrfs 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=conntrack 2024-09-17T01:25:24.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=cpu 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=diskstats 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=dmi 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=edac 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=entropy 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-09-17T01:25:24.637 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=filefd 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=filesystem 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=hwmon 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=infiniband 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=ipvs 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=loadavg 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=mdadm 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=meminfo 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=netclass 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=netdev 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=netstat 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=nfs 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=nfsd 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=nvme 2024-09-17T01:25:24.638 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=os 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=pressure 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=rapl 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=schedstat 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=sockstat 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=softnet 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=stat 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=tapestats 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=textfile 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=time 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=uname 2024-09-17T01:25:24.639 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=vmstat 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=xfs 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:115 level=info collector=zfs 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.211Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[40031]: ts=2024-09-17T01:25:24.212Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 bash[39956]: 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e 2024-09-17T01:25:24.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:25:24 smithi105 systemd[1]: Started Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:25:24.668 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T01:25:24.669 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T01:25:24.669 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T01:25:24.670 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T01:25:24.670 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T01:25:24.671 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T01:25:24.671 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T01:25:24.672 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T01:25:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4262750955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4262750955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:24 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/4262750955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:25:25.054 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:24 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:25.078 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.080 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.081 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.090 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.091 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.094 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:25.320 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:26 smithi105 ceph-mon[24602]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:27.405 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:26 smithi081 ceph-mon[23696]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:27.406 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:26 smithi081 ceph-mon[28146]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:28.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:28 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:28.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:28 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:28.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:28 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:29.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:29 smithi105 ceph-mon[24602]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:29 smithi081 ceph-mon[23696]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:29.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:29 smithi081 ceph-mon[28146]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:30.381 INFO:teuthology.orchestra.run.smithi081.stdout:77309411360 2024-09-17T01:25:30.381 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T01:25:31.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:31 smithi081 ceph-mon[23696]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:31.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:31 smithi081 ceph-mon[28146]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:31 smithi105 ceph-mon[24602]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:33.044 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:25:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:25:33.086 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:33 smithi081 ceph-mon[28146]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:33.087 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:33 smithi081 ceph-mon[23696]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:33.174 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:33 smithi105 ceph-mon[24602]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:33] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2024-09-17T01:25:34.368 INFO:teuthology.orchestra.run.smithi081.stdout:34359738411 2024-09-17T01:25:34.369 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T01:25:35.358 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:35 smithi081 ceph-mon[23696]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:35.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:35 smithi081 ceph-mon[28146]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:35 smithi105 ceph-mon[24602]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:36.651 INFO:teuthology.orchestra.run.smithi081.stdout:236223201292 2024-09-17T01:25:36.652 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T01:25:37.263 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:37 smithi081 ceph-mon[23696]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:37.263 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:37 smithi081 ceph-mon[28146]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:37 smithi105 ceph-mon[24602]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:38.489 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:38.498 INFO:teuthology.orchestra.run.smithi081.stdout:167503724566 2024-09-17T01:25:38.498 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T01:25:38.812 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:38.812 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:38.813 INFO:teuthology.orchestra.run.smithi081.stdout:55834574889 2024-09-17T01:25:38.813 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T01:25:39.367 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:39 smithi081 ceph-mon[23696]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:39.368 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:39 smithi081 ceph-mon[28146]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:39 smithi105 ceph-mon[24602]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:40.340 INFO:teuthology.orchestra.run.smithi081.stdout:133143986203 2024-09-17T01:25:40.341 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T01:25:41.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:41 smithi081 ceph-mon[23696]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:41.198 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:41 smithi081 ceph-mon[28146]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:41 smithi105 ceph-mon[24602]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:42.135 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:42.470 INFO:teuthology.orchestra.run.smithi081.stdout:111669149728 2024-09-17T01:25:42.470 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T01:25:43.057 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:25:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:25:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:43 smithi105 ceph-mon[24602]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:43 smithi081 ceph-mon[23696]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:43 smithi081 ceph-mon[28146]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:43] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2024-09-17T01:25:44.703 INFO:teuthology.orchestra.run.smithi081.stdout:201863462930 2024-09-17T01:25:44.704 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T01:25:45.320 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:45 smithi081 ceph-mon[23696]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:45.321 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:45 smithi081 ceph-mon[28146]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:45.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:45 smithi105 ceph-mon[24602]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:46.079 INFO:teuthology.orchestra.run.smithi081.stdout:236223201293 2024-09-17T01:25:46.664 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:47.305 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:47 smithi081 ceph-mon[23696]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:47.306 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1632178250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T01:25:47.306 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:47 smithi081 ceph-mon[28146]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:47.306 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1632178250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T01:25:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:47 smithi105 ceph-mon[24602]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1632178250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T01:25:47.959 INFO:teuthology.orchestra.run.smithi081.stdout:34359738413 2024-09-17T01:25:48.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:48 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4282430616' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T01:25:48.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:48 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4282430616' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T01:25:48.171 INFO:teuthology.orchestra.run.smithi081.stdout:167503724568 2024-09-17T01:25:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:48 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/4282430616' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T01:25:49.117 INFO:teuthology.orchestra.run.smithi081.stdout:77309411364 2024-09-17T01:25:49.117 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:49 smithi081 ceph-mon[28146]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:49.117 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:49 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1517677240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T01:25:49.117 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:49 smithi081 ceph-mon[23696]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:49.118 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:49 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1517677240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T01:25:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:49 smithi105 ceph-mon[24602]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:49 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1517677240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T01:25:49.652 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738411 got 34359738413 for osd.0 2024-09-17T01:25:49.652 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:50 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1562788577' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T01:25:50.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:50 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1562788577' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T01:25:50.299 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:50 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1562788577' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T01:25:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[23696]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[28146]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:51 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:51 smithi105 ceph-mon[24602]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:51 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:51 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:25:51.655 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724568 for osd.5 2024-09-17T01:25:51.655 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:52.094 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411360 got 77309411364 for osd.2 2024-09-17T01:25:52.095 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:52.304 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:52 smithi081 ceph-mon[23696]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:25:52.304 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:52 smithi081 ceph-mon[28146]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:25:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:52 smithi105 ceph-mon[24602]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:25:52.914 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:53.086 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:25:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:25:53.327 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:53 smithi081 ceph-mon[23696]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:53.328 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:53 smithi081 ceph-mon[28146]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:53.354 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201292 got 236223201293 for osd.7 2024-09-17T01:25:53.355 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:53 smithi105 ceph-mon[24602]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:53.748 INFO:teuthology.orchestra.run.smithi081.stdout:55834574892 2024-09-17T01:25:53.807 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:25:53.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:25:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:25:53] "GET /metrics HTTP/1.1" 200 191477 "" "Prometheus/2.33.4" 2024-09-17T01:25:54.377 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:54 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2856222394' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T01:25:54.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:54 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2856222394' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T01:25:54.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:54 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2856222394' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T01:25:55.364 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:55 smithi081 ceph-mon[23696]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:55.364 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:55 smithi081 ceph-mon[28146]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:55 smithi105 ceph-mon[24602]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:56.682 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574892 for osd.1 2024-09-17T01:25:56.682 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:57 smithi105 ceph-mon[24602]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:57 smithi081 ceph-mon[23696]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:57 smithi081 ceph-mon[28146]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:58.171 INFO:teuthology.orchestra.run.smithi081.stdout:133143986207 2024-09-17T01:25:59.346 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986207 for osd.4 2024-09-17T01:25:59.347 DEBUG:teuthology.parallel:result is None 2024-09-17T01:25:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:59 smithi105 ceph-mon[24602]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:25:59 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2462413304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T01:25:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:59 smithi081 ceph-mon[23696]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:25:59 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2462413304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T01:25:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:59 smithi081 ceph-mon[28146]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:25:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:25:59 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2462413304' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T01:26:00.296 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:26:00.343 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:26:00.343 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:26:00.344 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:26:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:26:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:26:01.267 INFO:teuthology.orchestra.run.smithi081.stdout:111669149731 2024-09-17T01:26:01.267 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:01 smithi081 ceph-mon[23696]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:01.267 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:01 smithi081 ceph-mon[28146]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:01 smithi105 ceph-mon[24602]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:02.366 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:02 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/332864260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T01:26:02.366 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:02 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/332864260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T01:26:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:02 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/332864260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T01:26:03.110 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:26:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:26:03.279 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149731 for osd.3 2024-09-17T01:26:03.279 DEBUG:teuthology.parallel:result is None 2024-09-17T01:26:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:03 smithi105 ceph-mon[24602]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:03.417 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:03 smithi081 ceph-mon[23696]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:03.417 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:03 smithi081 ceph-mon[28146]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:03.417 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:03 smithi081 systemd[1]: Starting Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:26:03.710 INFO:teuthology.orchestra.run.smithi081.stdout:201863462934 2024-09-17T01:26:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:26:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:03] "GET /metrics HTTP/1.1" 200 191481 "" "Prometheus/2.33.4" 2024-09-17T01:26:03.930 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:03 smithi081 podman[46023]: 2024-09-17 01:26:03.728480538 +0000 UTC m=+0.106574059 volume create d2cc6a8e3f24d95216b57d2d07c2eb542474bfdd2722c9817e7622ad8269e2fc 2024-09-17T01:26:03.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:03 smithi081 podman[46023]: 2024-09-17 01:26:03.63793043 +0000 UTC m=+0.016023956 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-17T01:26:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:04 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3189618007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T01:26:04.389 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3189618007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T01:26:04.389 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3189618007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T01:26:04.389 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 podman[46023]: 2024-09-17 01:26:04.07196577 +0000 UTC m=+0.450059329 container create 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:04.743 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462934 for osd.6 2024-09-17T01:26:04.743 DEBUG:teuthology.parallel:result is None 2024-09-17T01:26:04.744 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T01:26:04.744 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T01:26:04.984 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:26:05.005 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 podman[46023]: 2024-09-17 01:26:04.836109549 +0000 UTC m=+1.214203072 container init 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 podman[46023]: 2024-09-17 01:26:04.839178506 +0000 UTC m=+1.217272023 container start 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.852Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.852Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.854Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.81 port=9094 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.855Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.881Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.881Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.883Z caller=main.go:518 msg=Listening address=:9093 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:04.883Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 bash[46023]: 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 2024-09-17T01:26:05.006 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:04 smithi081 systemd[1]: Started Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:05.328 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:05.329 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:06.258 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:06 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:06.258 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:06 smithi081 ceph-mon[23696]: Deploying daemon grafana.a on smithi105 2024-09-17T01:26:06.258 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:06 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:06.258 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:06 smithi081 ceph-mon[28146]: Deploying daemon grafana.a on smithi105 2024-09-17T01:26:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:06 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:26:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:06 smithi105 ceph-mon[24602]: Deploying daemon grafana.a on smithi105 2024-09-17T01:26:06.649 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:26:06.650 INFO:teuthology.orchestra.run.smithi081.stderr:dumped all 2024-09-17T01:26:06.930 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:06.855Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000036634s 2024-09-17T01:26:07.338 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:07 smithi081 ceph-mon[23696]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:07.338 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:07 smithi081 ceph-mon[28146]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:07.363 INFO:teuthology.orchestra.run.smithi081.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2024-09-17T01:26:05.677405+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002034"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-17T01:24:59.741800+0000","last_change":"2024-09-17T01:24:48.002201+0000","last_active":"2024-09-17T01:24:59.741800+0000","last_peered":"2024-09-17T01:24:59.741800+0000","last_clean":"2024-09-17T01:24:59.741800+0000","last_became_active":"2024-09-17T01:24:48.001763+0000","last_became_peered":"2024-09-17T01:24:48.001763+0000","last_unstale":"2024-09-17T01:24:59.741800+0000","last_undegraded":"2024-09-17T01:24:59.741800+0000","last_fullsized":"2024-09-17T01:24:59.741800+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T01:22:50.976388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T01:22:50.976388+0000","last_clean_scrub_stamp":"2024-09-17T01:22:50.976388+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T12:04:55.705164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201297,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.65200000000000002}]},{"osd":1,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.58299999999999996}]},{"osd":2,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.73199999999999998}]},{"osd":3,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.63900000000000001}]},{"osd":4,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.439}]},{"osd":5,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.499},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.46000000000000002}]},{"osd":6,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.54800000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462934,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52900000000000003}]},{"osd":1,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.441},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.497}]},{"osd":2,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.48499999999999999}]},{"osd":3,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.42899999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.372},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.38300000000000001}]},{"osd":5,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.46899999999999997}]},{"osd":7,"last update":"Tue Sep 17 01:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.60499999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.32600000000000001,"15min":0.30499999999999999},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.55900000000000005,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.34599999999999997,"15min":0.32200000000000001},"min":{"1min":0.23599999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.52700000000000002,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.32900000000000001}]},{"osd":2,"last update":"Tue Sep 17 01:25:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.35699999999999998},"min":{"1min":0.29799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.49299999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.371},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.34899999999999998,"15min":0.33500000000000002},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.45700000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.309}]},{"osd":3,"last update":"Tue Sep 17 01:25:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41099999999999998,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.29699999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:25:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79200000000000004,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.39000000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.439}]},{"osd":5,"last update":"Tue Sep 17 01:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51900000000000002}]},{"osd":6,"last update":"Tue Sep 17 01:25:25 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.434}]},{"osd":7,"last update":"Tue Sep 17 01:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.59899999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.35099999999999998},"min":{"1min":0.27700000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.49099999999999999,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.35699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52500000000000002,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38100000000000001}]},{"osd":2,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41199999999999998,"15min":0.41799999999999998},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41799999999999998,"15min":0.42499999999999999},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.55600000000000005,"5min":0.621,"15min":0.621},"last":0.32100000000000001}]},{"osd":3,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.45000000000000001,"15min":0.45700000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.59399999999999997,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.47099999999999997,"15min":0.47499999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.35899999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.74399999999999999}]},{"osd":5,"last update":"Tue Sep 17 01:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.502}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.66300000000000003}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.70199999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.37,"15min":0.36499999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.505,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.26200000000000001}]},{"osd":1,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.378,"15min":0.378},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55300000000000005,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.40799999999999997,"15min":0.40999999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.249}]},{"osd":3,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.45800000000000002,"15min":0.46600000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.33100000000000002}]},{"osd":4,"last update":"Tue Sep 17 01:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.84599999999999997}]},{"osd":5,"last update":"Tue Sep 17 01:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.63}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.88},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.89400000000000002}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.73399999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.40100000000000002,"15min":0.40300000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.55400000000000005,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.61099999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.55200000000000005,"5min":0.621,"15min":0.621},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.44400000000000001,"15min":0.44700000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.623}]},{"osd":2,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.55000000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.27700000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.59299999999999997}]},{"osd":4,"last update":"Tue Sep 17 01:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73999999999999999,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67300000000000004}]},{"osd":5,"last update":"Tue Sep 17 01:26:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76500000000000001,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.68899999999999995}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.81799999999999995}]},{"osd":7,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.80500000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5764,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739388,"statfs":{"total":95995035648,"available":95989133312,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.61399999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.56699999999999995,"15min":0.57699999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.64100000000000001,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.46899999999999997}]},{"osd":1,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.41799999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.71599999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.66500000000000004,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.435}]},{"osd":2,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54700000000000004,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.61399999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.502},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.56799999999999995,"15min":0.57499999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.63800000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.40999999999999998}]},{"osd":3,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.57399999999999995,"15min":0.58299999999999996},"min":{"1min":0.40000000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.65700000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.49399999999999999}]},{"osd":5,"last update":"Tue Sep 17 01:25:00 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.36299999999999999}]},{"osd":6,"last update":"Tue Sep 17 01:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.39400000000000002}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724571,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.502,"15min":0.50700000000000001},"min":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.621,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68899999999999995,"5min":0.748,"15min":0.748},"last":0.57199999999999995}]},{"osd":1,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64800000000000002,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.61199999999999999,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":8.6010000000000009},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.59999999999999998,"15min":0.55600000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.6300000000000008,"5min":8.6300000000000008,"15min":8.6300000000000008},"last":0.50700000000000001}]},{"osd":3,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.61099999999999999,"15min":0.56599999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.5899999999999999,"5min":8.5899999999999999,"15min":8.5899999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58199999999999996,"15min":0.58599999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.69399999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.64200000000000002}]},{"osd":4,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.39400000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39500000000000002}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.436}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T01:26:07.364 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T01:26:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:07 smithi105 ceph-mon[24602]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:07.569 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:26:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:08 smithi105 ceph-mon[24602]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:08.403 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:08 smithi081 ceph-mon[23696]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:08.403 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:08 smithi081 ceph-mon[28146]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:09 smithi105 ceph-mon[24602]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:09.409 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:26:09.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:09 smithi081 ceph-mon[23696]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:09.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:09 smithi081 ceph-mon[28146]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:09.412 INFO:teuthology.orchestra.run.smithi081.stderr:dumped all 2024-09-17T01:26:09.988 INFO:teuthology.orchestra.run.smithi081.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-09-17T01:26:07.677761+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002128"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-17T01:24:59.741800+0000","last_change":"2024-09-17T01:24:48.002201+0000","last_active":"2024-09-17T01:24:59.741800+0000","last_peered":"2024-09-17T01:24:59.741800+0000","last_clean":"2024-09-17T01:24:59.741800+0000","last_became_active":"2024-09-17T01:24:48.001763+0000","last_became_peered":"2024-09-17T01:24:48.001763+0000","last_unstale":"2024-09-17T01:24:59.741800+0000","last_undegraded":"2024-09-17T01:24:59.741800+0000","last_fullsized":"2024-09-17T01:24:59.741800+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T01:22:50.976388+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T01:22:50.976388+0000","last_clean_scrub_stamp":"2024-09-17T01:22:50.976388+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T12:04:55.705164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.60399999999999998}]},{"osd":1,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.66200000000000003}]},{"osd":2,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.628}]},{"osd":3,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.65400000000000003}]},{"osd":4,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.46800000000000003}]},{"osd":5,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.497}]},{"osd":6,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.47899999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64900000000000002}]},{"osd":1,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.54700000000000004}]},{"osd":2,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.60099999999999998}]},{"osd":3,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63400000000000001}]},{"osd":4,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.496}]},{"osd":5,"last update":"Tue Sep 17 01:25:21 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.46200000000000002}]},{"osd":7,"last update":"Tue Sep 17 01:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.56200000000000006}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.32600000000000001,"15min":0.30499999999999999},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.55900000000000005,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.34599999999999997,"15min":0.32200000000000001},"min":{"1min":0.23599999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.52700000000000002,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.52200000000000002}]},{"osd":2,"last update":"Tue Sep 17 01:25:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.377,"15min":0.35699999999999998},"min":{"1min":0.29799999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.49299999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.34899999999999998,"15min":0.33500000000000002},"min":{"1min":0.24399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":0.45700000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.503}]},{"osd":3,"last update":"Tue Sep 17 01:25:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41099999999999998,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Tue Sep 17 01:25:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79200000000000004,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56100000000000005,"15min":0.55700000000000005},"min":{"1min":0.39000000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.61199999999999999}]},{"osd":5,"last update":"Tue Sep 17 01:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.45100000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79600000000000004,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.59399999999999997,"15min":0.59299999999999997},"min":{"1min":0.45000000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.628}]},{"osd":6,"last update":"Tue Sep 17 01:25:25 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.66900000000000004}]},{"osd":7,"last update":"Tue Sep 17 01:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.72599999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36299999999999999,"15min":0.35099999999999998},"min":{"1min":0.27700000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.49099999999999999,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.35699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52500000000000002,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.60699999999999998}]},{"osd":2,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.41199999999999998,"15min":0.41799999999999998},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.41799999999999998,"15min":0.42499999999999999},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.55600000000000005,"5min":0.621,"15min":0.621},"last":0.621}]},{"osd":3,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.45000000000000001,"15min":0.45700000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.59399999999999997,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.47099999999999997,"15min":0.47499999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.65200000000000002}]},{"osd":4,"last update":"Tue Sep 17 01:25:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.81699999999999995}]},{"osd":5,"last update":"Tue Sep 17 01:26:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57399999999999995,"15min":0.57499999999999996},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.84299999999999997}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.876},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.88600000000000001}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.871}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.37,"15min":0.36499999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.505,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.34999999999999998}]},{"osd":1,"last update":"Tue Sep 17 01:25:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.378,"15min":0.378},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55300000000000005,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.379,"5min":0.40799999999999997,"15min":0.40999999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.33600000000000002}]},{"osd":3,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.45800000000000002,"15min":0.46600000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.30299999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.58299999999999996}]},{"osd":5,"last update":"Tue Sep 17 01:26:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.40400000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66200000000000003}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.63100000000000001}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.61699999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.40100000000000002,"15min":0.40300000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.55400000000000005,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.45700000000000002}]},{"osd":1,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.55200000000000005,"5min":0.621,"15min":0.621},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.44400000000000001,"15min":0.44700000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.44600000000000001}]},{"osd":2,"last update":"Tue Sep 17 01:25:16 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.55000000000000004,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.27700000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.47499999999999998}]},{"osd":4,"last update":"Tue Sep 17 01:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73999999999999999,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.81200000000000006}]},{"osd":5,"last update":"Tue Sep 17 01:26:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76500000000000001,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.91600000000000004}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.69399999999999995}]},{"osd":7,"last update":"Tue Sep 17 01:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.749},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.875}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5764,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739388,"statfs":{"total":95995035648,"available":95989133312,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53700000000000003,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.61399999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.56699999999999995,"15min":0.57699999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.64100000000000001,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52200000000000002}]},{"osd":1,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.41799999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.71599999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.66500000000000004,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.435}]},{"osd":2,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54700000000000004,"15min":0.55400000000000005},"min":{"1min":0.41699999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.61399999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.56799999999999995,"15min":0.57499999999999996},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.63800000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.48399999999999999}]},{"osd":3,"last update":"Tue Sep 17 01:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.57399999999999995,"15min":0.58299999999999996},"min":{"1min":0.40000000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.65700000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.501},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56499999999999995,"15min":0.57299999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.65800000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.46400000000000002}]},{"osd":5,"last update":"Tue Sep 17 01:26:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.33500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.44500000000000001,"15min":0.439},"min":{"1min":0.30399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.36399999999999999}]},{"osd":6,"last update":"Tue Sep 17 01:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.52900000000000003}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.45700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.502,"15min":0.50700000000000001},"min":{"1min":0.34799999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.621,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68899999999999995,"5min":0.748,"15min":0.748},"last":0.54300000000000004}]},{"osd":1,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64800000000000002,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.47699999999999998}]},{"osd":2,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.61199999999999999,"15min":0.56799999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.6010000000000009,"5min":8.6010000000000009,"15min":8.6010000000000009},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.59999999999999998,"15min":0.55600000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.6300000000000008,"5min":8.6300000000000008,"15min":8.6300000000000008},"last":0.438}]},{"osd":3,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.61099999999999999,"15min":0.56599999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.5899999999999999,"5min":8.5899999999999999,"15min":8.5899999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58199999999999996,"15min":0.58599999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.69399999999999995,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.58399999999999996}]},{"osd":4,"last update":"Tue Sep 17 01:25:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.373},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.39600000000000002,"15min":0.39400000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.51100000000000001}]},{"osd":6,"last update":"Tue Sep 17 01:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.52600000000000002}]},{"osd":7,"last update":"Tue Sep 17 01:25:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.5},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T01:26:09.990 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T01:26:09.990 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T01:26:09.990 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T01:26:09.990 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T01:26:10.199 INFO:teuthology.orchestra.run.smithi081.stderr:Inferring config /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T01:26:10.319 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:10 smithi081 ceph-mon[23696]: from='client.24499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:10.319 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:10 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:10.320 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:10 smithi081 ceph-mon[28146]: from='client.24499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:10.320 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:10 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:10 smithi105 ceph-mon[24602]: from='client.24499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:26:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:10 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:11 smithi081 ceph-mon[23696]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:11 smithi081 ceph-mon[28146]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:11 smithi105 ceph-mon[24602]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:11.864 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:26:11.864 INFO:teuthology.orchestra.run.smithi081.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T01:26:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:12 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2992757455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T01:26:12.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:12 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2992757455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T01:26:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:12 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2992757455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T01:26:12.702 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T01:26:12.703 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T01:26:12.703 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:26:12.712 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T01:26:12.713 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-09-17T01:26:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:26:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:26:13.337 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:13 smithi081 ceph-mon[23696]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:13.337 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:13 smithi081 ceph-mon[28146]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:13.462 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:13 smithi105 ceph-mon[24602]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:26:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:13] "GET /metrics HTTP/1.1" 200 191481 "" "Prometheus/2.33.4" 2024-09-17T01:26:15.048 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:14.857Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002137114s 2024-09-17T01:26:15.348 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:15 smithi081 ceph-mon[23696]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:15.348 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:15 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' 2024-09-17T01:26:15.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:15 smithi081 ceph-mon[28146]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:15.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:15 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' 2024-09-17T01:26:15.369 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:26:15.378 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T01:26:15.378 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-09-17T01:26:15.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:15 smithi105 ceph-mon[24602]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:15.438 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:15 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' 2024-09-17T01:26:16.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-17T01:26:16.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-17T01:26:17.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-17T01:26:17.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-17T01:26:17.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-09-17T01:26:17.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-17T01:26:17.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-09-17T01:26:17.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-09-17T01:26:17.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-17T01:26:17.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-17T01:26:17.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:16+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-17T01:26:17.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-09-17T01:26:17.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-17T01:26:17.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-09-17T01:26:17.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-09-17T01:26:17.095 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-09-17T01:26:17.096 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-17T01:26:17.097 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-09-17T01:26:17.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-09-17T01:26:17.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-09-17T01:26:17.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-09-17T01:26:17.345 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-09-17T01:26:17.346 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-09-17T01:26:17.347 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-09-17T01:26:17.348 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=535.531821ms 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-17T01:26:17.349 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T01:26:17.350 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:26:17+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-17T01:26:17.350 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 bash[40694]: d2a64aee5705c20cc4afe53bc35d51383502ab3f10874d6ae0580ba0fc0417a2 2024-09-17T01:26:17.350 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:26:17 smithi105 systemd[1]: Started Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:17.612 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:17 smithi105 ceph-mon[24602]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:17.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:17 smithi081 ceph-mon[23696]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:17.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:17 smithi081 ceph-mon[28146]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "13126dd2-13e7-4bff-9903-58d037fe238b", 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "r", 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout: "current_period": "19a11dd9-4638-4431-8458-8e2b97c244ba", 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout: "epoch": 1 2024-09-17T01:26:18.436 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:26:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:18 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:18 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:18 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:18 smithi105 ceph-mon[24602]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T01:26:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:18 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T01:26:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[23696]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[28146]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T01:26:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:18 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T01:26:19.096 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-09-17T01:26:19.621 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:19 smithi105 ceph-mon[24602]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:19.621 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:19 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T01:26:19.621 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:19 smithi105 ceph-mon[24602]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T01:26:19.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[28146]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:19.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T01:26:19.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[28146]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T01:26:19.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[23696]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:19.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3091394316' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T01:26:19.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:19 smithi081 ceph-mon[23696]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T01:26:20.632 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:20 smithi081 ceph-mon[23696]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T01:26:20.632 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:20 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:20.632 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:20 smithi081 ceph-mon[28146]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T01:26:20.632 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:20 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:20.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:20 smithi105 ceph-mon[24602]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T01:26:20.807 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:20 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "57867236-6eab-4dc9-b578-f6a8f907e473", 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "default", 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "api_name": "default", 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "is_master": "true", 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "endpoints": [], 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "hostnames": [], 2024-09-17T01:26:21.694 INFO:teuthology.orchestra.run.smithi081.stdout: "hostnames_s3website": [], 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "master_zone": "", 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "zones": [], 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "placement_targets": [], 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "default_placement": "", 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_id": "13126dd2-13e7-4bff-9903-58d037fe238b", 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "sync_policy": { 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: "groups": [] 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:21.695 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:26:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:21 smithi105 ceph-mon[24602]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:21 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:21.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:21 smithi081 ceph-mon[23696]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:21.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:21 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:21.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:21 smithi081 ceph-mon[28146]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:21.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:21 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.075 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-09-17T01:26:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:26:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:26:23.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[23696]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:23.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.678 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[28146]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:23.678 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.678 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:23 smithi105 ceph-mon[24602]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:23 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:23 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:26:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:23] "GET /metrics HTTP/1.1" 200 191484 "" "Prometheus/2.33.4" 2024-09-17T01:26:24.761 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:24 smithi081 ceph-mon[23696]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:26:24.761 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:24 smithi081 ceph-mon[23696]: Reconfiguring daemon alertmanager.a on smithi081 2024-09-17T01:26:24.762 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:24 smithi081 ceph-mon[28146]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:26:24.762 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:24 smithi081 ceph-mon[28146]: Reconfiguring daemon alertmanager.a on smithi081 2024-09-17T01:26:24.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:24 smithi105 ceph-mon[24602]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:26:24.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:24 smithi105 ceph-mon[24602]: Reconfiguring daemon alertmanager.a on smithi081 2024-09-17T01:26:25.809 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:25 smithi105 ceph-mon[24602]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:25.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:25 smithi081 ceph-mon[23696]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:25.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:25 smithi081 ceph-mon[28146]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-09-17T01:26:26.601 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:26 smithi081 systemd[1]: Stopping Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:26:26.601 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:26 smithi081 bash[47605]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-17T01:26:26.925 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[46054]: level=info ts=2024-09-17T01:26:26.603Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:26:27.178 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:26 smithi081 podman[47616]: 2024-09-17 01:26:26.962859476 +0000 UTC m=+0.463131570 container died 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:27.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:27 smithi081 podman[47616]: 2024-09-17 01:26:27.271109194 +0000 UTC m=+0.771381290 container cleanup 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:27.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:27 smithi081 bash[47616]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a 2024-09-17T01:26:27.721 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:27 smithi081 ceph-mon[23696]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-17T01:26:27.721 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:27 smithi081 ceph-mon[28146]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-17T01:26:27.722 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:27 smithi081 podman[47629]: 2024-09-17 01:26:27.472122972 +0000 UTC m=+0.838980658 container remove 43112c82d52e79fde9f41f7a9a8df52e7c0f3bf830a1dac86ca232a27af08316 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:27.722 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:27 smithi081 podman[47629]: 2024-09-17 01:26:27.547705559 +0000 UTC m=+0.914563252 volume remove d2cc6a8e3f24d95216b57d2d07c2eb542474bfdd2722c9817e7622ad8269e2fc 2024-09-17T01:26:27.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:27 smithi105 ceph-mon[24602]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "5c375633-ed87-4def-a68d-55f2758f14f9", 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "z", 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout: "domain_root": "z.rgw.meta:root", 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout: "control_pool": "z.rgw.control", 2024-09-17T01:26:28.065 INFO:teuthology.orchestra.run.smithi081.stdout: "gc_pool": "z.rgw.log:gc", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "lc_pool": "z.rgw.log:lc", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "log_pool": "z.rgw.log", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "roles_pool": "z.rgw.meta:roles", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "otp_pool": "z.rgw.otp", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "system_key": { 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "access_key": "", 2024-09-17T01:26:28.066 INFO:teuthology.orchestra.run.smithi081.stdout: "secret_key": "" 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "placement_pools": [ 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: { 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "key": "default-placement", 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "val": { 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "index_pool": "z.rgw.buckets.index", 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "storage_classes": { 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "STANDARD": { 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "data_pool": "z.rgw.buckets.data" 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:28.067 INFO:teuthology.orchestra.run.smithi081.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: "index_type": 0 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_id": "13126dd2-13e7-4bff-9903-58d037fe238b", 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout: "notif_pool": "z.rgw.log:notif" 2024-09-17T01:26:28.068 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:26:28.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 bash[47661]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager.a" found: no such container 2024-09-17T01:26:28.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T01:26:28.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 systemd[1]: Stopped Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:28.431 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 systemd[1]: Starting Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:26:28.576 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-09-17T01:26:28.886 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 podman[47827]: 2024-09-17 01:26:28.780605212 +0000 UTC m=+0.110527692 volume create 20151da017a9e094ad6353c970a4ff36ba592a1ffac1ebf5690475fe8b055bad 2024-09-17T01:26:28.886 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 podman[47827]: 2024-09-17 01:26:28.685762498 +0000 UTC m=+0.015684961 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-09-17T01:26:28.887 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:28 smithi081 podman[47827]: 2024-09-17 01:26:28.889276059 +0000 UTC m=+0.219198494 container create 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 podman[47827]: 2024-09-17 01:26:29.183939792 +0000 UTC m=+0.513862247 container init 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 podman[47827]: 2024-09-17 01:26:29.186523965 +0000 UTC m=+0.516446409 container start 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.197Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.197Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.198Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.81 port=9094 2024-09-17T01:26:29.434 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.199Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.223Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.223Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.226Z caller=main.go:518 msg=Listening address=:9093 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:29.226Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 bash[47827]: 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba 2024-09-17T01:26:29.435 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:29 smithi081 systemd[1]: Started Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:29 smithi105 ceph-mon[24602]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 441 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-17T01:26:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:29 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:29.742 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-mon[23696]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 441 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-17T01:26:29.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:29.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-mon[28146]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 441 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-09-17T01:26:29.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:29 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:30.724 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:30 smithi081 ceph-mon[23696]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:26:30.724 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:30 smithi081 ceph-mon[23696]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:26:30.724 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:30 smithi081 ceph-mon[28146]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:26:30.724 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:30 smithi081 ceph-mon[28146]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:26:30.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:30 smithi105 ceph-mon[24602]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:26:30.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:30 smithi105 ceph-mon[24602]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:26:31.574 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:31.199Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000250697s 2024-09-17T01:26:31.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:31 smithi105 ceph-mon[24602]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 399 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-09-17T01:26:31.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:31 smithi081 ceph-mon[23696]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 399 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-09-17T01:26:31.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:31 smithi081 ceph-mon[28146]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 399 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-09-17T01:26:32.505 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:26:32.505 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 bash[41371]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-17T01:26:32.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-mon[24602]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T01:26:32.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T01:26:32.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-mon[24602]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T01:26:32.789 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.507Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[39729]: ts=2024-09-17T01:26:32.508Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-17T01:26:32.790 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 podman[41381]: 2024-09-17 01:26:32.518998745 +0000 UTC m=+0.119276400 container died 87b69b3b6bd39ccc802d7597c6535f6c7712fe43b2b99799dcac712695e10727 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[28146]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T01:26:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[28146]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[23696]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T01:26:32.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:32 smithi081 ceph-mon[23696]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T01:26:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 podman[41381]: 2024-09-17 01:26:32.898971679 +0000 UTC m=+0.499249333 container cleanup 87b69b3b6bd39ccc802d7597c6535f6c7712fe43b2b99799dcac712695e10727 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:33.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 bash[41381]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T01:26:33.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:32 smithi105 bash[41409]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus.a" found: no such container 2024-09-17T01:26:33.389 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 podman[41395]: 2024-09-17 01:26:33.14162801 +0000 UTC m=+0.623843697 container remove 87b69b3b6bd39ccc802d7597c6535f6c7712fe43b2b99799dcac712695e10727 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:33.390 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T01:26:33.390 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 systemd[1]: Stopped Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:33.390 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 systemd[1]: Starting Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:26:33.390 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 podman[41471]: 2024-09-17 01:26:33.274980311 +0000 UTC m=+0.017056734 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-09-17T01:26:33.667 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-mon[24602]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 819 B/s wr, 1 op/s 2024-09-17T01:26:33.667 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 podman[41471]: 2024-09-17 01:26:33.392526036 +0000 UTC m=+0.134602448 container create 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:33.667 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 podman[41471]: 2024-09-17 01:26:33.594831337 +0000 UTC m=+0.336907759 container init 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:33.667 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 podman[41471]: 2024-09-17 01:26:33.597819111 +0000 UTC m=+0.339895536 container start 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:26:33.667 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T01:26:33.667 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:518 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.621Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.623Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.623Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.624Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.626Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.626Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.34µs 2024-09-17T01:26:33.668 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.626Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T01:26:33.669 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 bash[41471]: 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b 2024-09-17T01:26:33.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:33 smithi081 ceph-mon[23696]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 819 B/s wr, 1 op/s 2024-09-17T01:26:33.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:33 smithi081 ceph-mon[28146]: pgmap v53: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 819 B/s wr, 1 op/s 2024-09-17T01:26:34.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:26:34.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.829Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-17T01:26:34.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.830Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-17T01:26:34.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.830Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.409µs wal_replay_duration=203.202043ms total_replay_duration=203.26069ms 2024-09-17T01:26:34.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.832Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T01:26:34.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.833Z caller=main.go:947 level=info msg="TSDB started" 2024-09-17T01:26:34.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.833Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T01:26:34.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.857Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.722517ms db_storage=805ns remote_storage=1.66µs web_handler=477ns query_engine=810ns scrape=576.976µs scrape_sd=34.092µs notify=29.58µs notify_sd=12.081µs rules=23.683468ms 2024-09-17T01:26:34.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:26:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:26:33.857Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-09-17T01:26:34.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T01:26:34.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:34.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:34.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T01:26:34.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:34 smithi105 ceph-mon[24602]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T01:26:34.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T01:26:34.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:34.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[23696]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3039753434' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:34.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T01:26:34.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:34 smithi081 ceph-mon[28146]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T01:26:35.867 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: pgmap v56: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 853 B/s wr, 1 op/s 2024-09-17T01:26:35.867 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:35.868 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:35 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:35.925 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: pgmap v56: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 853 B/s wr, 1 op/s 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:35.926 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: pgmap v56: 97 pgs: 64 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 853 B/s wr, 1 op/s 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.81:9093"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:35.927 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:35 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T01:26:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: pgmap v59: 129 pgs: 96 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: pgmap v59: 129 pgs: 96 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:37 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: pgmap v59: 129 pgs: 96 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T01:26:37.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T01:26:37.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:37.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:37 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T01:26:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T01:26:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T01:26:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:38 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.569 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2710705676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:38 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "8e5cf2ac-7873-4a7b-b5bc-ceefedcad6a7", 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "epoch": 1, 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "predecessor_uuid": "19a11dd9-4638-4431-8458-8e2b97c244ba", 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "sync_status": [], 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "period_map": { 2024-09-17T01:26:38.656 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "8e5cf2ac-7873-4a7b-b5bc-ceefedcad6a7", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "zonegroups": [ 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: { 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "57867236-6eab-4dc9-b578-f6a8f907e473", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "default", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "api_name": "default", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "is_master": "true", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "endpoints": [], 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "hostnames": [], 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "hostnames_s3website": [], 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "master_zone": "5c375633-ed87-4def-a68d-55f2758f14f9", 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "zones": [ 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: { 2024-09-17T01:26:38.657 INFO:teuthology.orchestra.run.smithi081.stdout: "id": "5c375633-ed87-4def-a68d-55f2758f14f9", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "z", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "endpoints": [], 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "log_meta": "false", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "log_data": "false", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "bucket_index_max_shards": 11, 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "read_only": "false", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "tier_type": "", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "sync_from_all": "true", 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "sync_from": [], 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "redirect_zone": "" 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: "placement_targets": [ 2024-09-17T01:26:38.658 INFO:teuthology.orchestra.run.smithi081.stdout: { 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "name": "default-placement", 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "tags": [], 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "storage_classes": [ 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "STANDARD" 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: ] 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "default_placement": "default-placement", 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_id": "13126dd2-13e7-4bff-9903-58d037fe238b", 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "sync_policy": { 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: "groups": [] 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.659 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "short_zone_ids": [ 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: { 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "key": "5c375633-ed87-4def-a68d-55f2758f14f9", 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "val": 486552628 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: ] 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "master_zonegroup": "57867236-6eab-4dc9-b578-f6a8f907e473", 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "master_zone": "5c375633-ed87-4def-a68d-55f2758f14f9", 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "period_config": { 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "bucket_quota": { 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "enabled": false, 2024-09-17T01:26:38.660 INFO:teuthology.orchestra.run.smithi081.stdout: "check_on_raw": false, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_size": -1, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_size_kb": 0, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_objects": -1 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "user_quota": { 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "enabled": false, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "check_on_raw": false, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_size": -1, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_size_kb": 0, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_objects": -1 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "user_ratelimit": { 2024-09-17T01:26:38.661 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_ops": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_ops": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_bytes": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_bytes": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "enabled": false 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "bucket_ratelimit": { 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_ops": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_ops": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_bytes": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_bytes": 0, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "enabled": false 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "anonymous_ratelimit": { 2024-09-17T01:26:38.662 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_ops": 0, 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_ops": 0, 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "max_read_bytes": 0, 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "max_write_bytes": 0, 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "enabled": false 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_id": "13126dd2-13e7-4bff-9903-58d037fe238b", 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_name": "r", 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout: "realm_epoch": 2 2024-09-17T01:26:38.663 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:26:39.371 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:26:39.202Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002341391s 2024-09-17T01:26:39.399 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-09-17T01:26:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[28146]: pgmap v62: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-17T01:26:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T01:26:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[28146]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:26:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[23696]: pgmap v62: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-17T01:26:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T01:26:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:39 smithi081 ceph-mon[23696]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:26:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:39 smithi105 ceph-mon[24602]: pgmap v62: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-09-17T01:26:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:39 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T01:26:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:39 smithi105 ceph-mon[24602]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:26:41.420 INFO:teuthology.orchestra.run.smithi081.stdout:Scheduled rgw.foo update... 2024-09-17T01:26:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:41 smithi081 ceph-mon[28146]: pgmap v64: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 499 B/s wr, 7 op/s 2024-09-17T01:26:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:41 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:41 smithi081 ceph-mon[23696]: pgmap v64: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 499 B/s wr, 7 op/s 2024-09-17T01:26:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:41 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:41.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:41 smithi105 ceph-mon[24602]: pgmap v64: 129 pgs: 32 creating+peering, 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 499 B/s wr, 7 op/s 2024-09-17T01:26:41.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:41 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:42.047 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph osd pool create foo' 2024-09-17T01:26:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:42 smithi105 ceph-mon[24602]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:42 smithi105 ceph-mon[24602]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:42.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:26:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:26:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:42 smithi081 ceph-mon[23696]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:42 smithi081 ceph-mon[23696]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:42.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:42 smithi081 ceph-mon[28146]: from='client.24577 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:42.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:42 smithi081 ceph-mon[28146]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:43.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:43 smithi105 ceph-mon[24602]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-17T01:26:43.895 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:43 smithi081 ceph-mon[23696]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-17T01:26:43.895 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:26:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:43] "GET /metrics HTTP/1.1" 200 193055 "" "Prometheus/2.33.4" 2024-09-17T01:26:43.896 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:43 smithi081 ceph-mon[28146]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-09-17T01:26:44.627 INFO:teuthology.orchestra.run.smithi081.stderr:pool 'foo' created 2024-09-17T01:26:44.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2252397606' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:44 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:44.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2252397606' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:44.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2252397606' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:44.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:44 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:45.348 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'rbd pool init foo' 2024-09-17T01:26:45.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:45 smithi105 ceph-mon[24602]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 1010 B/s wr, 2 op/s 2024-09-17T01:26:45.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:45 smithi105 ceph-mon[24602]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:45.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:45 smithi105 ceph-mon[24602]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:26:45.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:45 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T01:26:45.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:45 smithi105 ceph-mon[24602]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[23696]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 1010 B/s wr, 2 op/s 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[23696]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[23696]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[23696]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:26:45.928 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[28146]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 1010 B/s wr, 2 op/s 2024-09-17T01:26:45.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[28146]: Saving service rgw.foo spec with placement count:2 2024-09-17T01:26:45.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[28146]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:26:45.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T01:26:45.929 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:45 smithi081 ceph-mon[28146]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:26:46.876 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:46 smithi081 ceph-mon[23696]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T01:26:46.876 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:46 smithi081 ceph-mon[28146]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T01:26:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:46 smithi105 ceph-mon[24602]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T01:26:47.750 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:47 smithi081 ceph-mon[23696]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1011 B/s wr, 2 op/s 2024-09-17T01:26:47.750 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:47 smithi081 ceph-mon[23696]: osdmap e74: 8 total, 8 up, 8 in 2024-09-17T01:26:47.750 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:47 smithi081 ceph-mon[28146]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1011 B/s wr, 2 op/s 2024-09-17T01:26:47.750 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:47 smithi081 ceph-mon[28146]: osdmap e74: 8 total, 8 up, 8 in 2024-09-17T01:26:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:47 smithi105 ceph-mon[24602]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 1011 B/s wr, 2 op/s 2024-09-17T01:26:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:47 smithi105 ceph-mon[24602]: osdmap e74: 8 total, 8 up, 8 in 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1655693145' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:48 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:48.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1655693145' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:48.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1655693145' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:48.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:48 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[23696]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:49.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[23696]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:26:49.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[23696]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T01:26:49.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[23696]: osdmap e75: 8 total, 8 up, 8 in 2024-09-17T01:26:49.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[28146]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:49.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[28146]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:26:49.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[28146]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T01:26:49.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:49 smithi081 ceph-mon[28146]: osdmap e75: 8 total, 8 up, 8 in 2024-09-17T01:26:50.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:49 smithi105 ceph-mon[24602]: pgmap v71: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:50.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:49 smithi105 ceph-mon[24602]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:26:50.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:49 smithi105 ceph-mon[24602]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T01:26:50.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:49 smithi105 ceph-mon[24602]: osdmap e75: 8 total, 8 up, 8 in 2024-09-17T01:26:50.794 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:50 smithi105 ceph-mon[24602]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T01:26:50.794 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:50 smithi105 ceph-mon[24602]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:50 smithi081 ceph-mon[23696]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T01:26:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:50 smithi081 ceph-mon[23696]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:50.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:50 smithi081 ceph-mon[28146]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T01:26:50.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:50 smithi081 ceph-mon[28146]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:26:51.447 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch apply iscsi foo u p' 2024-09-17T01:26:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[23696]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T01:26:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[28146]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T01:26:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:51 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:52.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:51 smithi105 ceph-mon[24602]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T01:26:52.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:51 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:52.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:51 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:52.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:51 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:26:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:52 smithi105 ceph-mon[24602]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 6.9 KiB/s wr, 286 op/s 2024-09-17T01:26:52.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:26:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:26:52.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:52 smithi081 ceph-mon[23696]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 6.9 KiB/s wr, 286 op/s 2024-09-17T01:26:52.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:52 smithi081 ceph-mon[28146]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 6.9 KiB/s wr, 286 op/s 2024-09-17T01:26:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:26:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:26:53] "GET /metrics HTTP/1.1" 200 197880 "" "Prometheus/2.33.4" 2024-09-17T01:26:54.084 INFO:teuthology.orchestra.run.smithi081.stdout:Scheduled iscsi.foo update... 2024-09-17T01:26:54.866 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:26:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 5.8 KiB/s wr, 241 op/s 2024-09-17T01:26:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: Saving service iscsi.foo spec with placement count:1 2024-09-17T01:26:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 5.8 KiB/s wr, 241 op/s 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: Saving service iscsi.foo spec with placement count:1 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:55 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 108 KiB/s rd, 5.8 KiB/s wr, 241 op/s 2024-09-17T01:26:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: from='client.24634 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:26:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: Saving service iscsi.foo spec with placement count:1 2024-09-17T01:26:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:55.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:55 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.742 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 5.0 KiB/s wr, 286 op/s 2024-09-17T01:26:57.742 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: Checking dashboard <-> RGW credentials 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 5.0 KiB/s wr, 286 op/s 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: Checking dashboard <-> RGW credentials 2024-09-17T01:26:57.743 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.744 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.744 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.744 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:26:57.744 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T01:26:57.744 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:57 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 5.0 KiB/s wr, 286 op/s 2024-09-17T01:26:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: Checking dashboard <-> RGW credentials 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T01:26:57.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:57 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:26:58.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:58 smithi105 ceph-mon[24602]: Checking pool "foo" exists for service iscsi.foo 2024-09-17T01:26:58.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:58 smithi105 ceph-mon[24602]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:26:58.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:58 smithi081 ceph-mon[28146]: Checking pool "foo" exists for service iscsi.foo 2024-09-17T01:26:58.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:58 smithi081 ceph-mon[28146]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:26:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:58 smithi081 ceph-mon[23696]: Checking pool "foo" exists for service iscsi.foo 2024-09-17T01:26:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:58 smithi081 ceph-mon[23696]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:26:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:59 smithi081 ceph-mon[23696]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 211 KiB/s rd, 4.5 KiB/s wr, 378 op/s 2024-09-17T01:26:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:26:59 smithi081 ceph-mon[23696]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-17T01:26:59.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:59 smithi081 ceph-mon[28146]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 211 KiB/s rd, 4.5 KiB/s wr, 378 op/s 2024-09-17T01:26:59.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:26:59 smithi081 ceph-mon[28146]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-17T01:26:59.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:59 smithi105 ceph-mon[24602]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 211 KiB/s rd, 4.5 KiB/s wr, 378 op/s 2024-09-17T01:26:59.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:26:59 smithi105 ceph-mon[24602]: mgrmap e20: y(active, since 118s), standbys: x 2024-09-17T01:27:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-09-17T01:27:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-09-17T01:27:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:00.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:00.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:00.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[23696]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 169 KiB/s rd, 3.6 KiB/s wr, 304 op/s 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/959288594' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[23696]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3600241645' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:01.852 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[28146]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 169 KiB/s rd, 3.6 KiB/s wr, 304 op/s 2024-09-17T01:27:01.853 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/959288594' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:27:01.853 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[28146]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T01:27:01.853 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3600241645' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:01.853 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:01 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:01 smithi105 ceph-mon[24602]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 169 KiB/s rd, 3.6 KiB/s wr, 304 op/s 2024-09-17T01:27:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:01 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/959288594' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:27:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:01 smithi105 ceph-mon[24602]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T01:27:01.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:01 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3600241645' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:01.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:01 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]: dispatch 2024-09-17T01:27:03.037 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:03.315 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[23696]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 511 B/s wr, 235 op/s 2024-09-17T01:27:03.315 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]': finished 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[23696]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/53756042' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[28146]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 511 B/s wr, 235 op/s 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]': finished 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[28146]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/53756042' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.316 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:03 smithi105 ceph-mon[24602]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 511 B/s wr, 235 op/s 2024-09-17T01:27:03.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:03 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3298429670"}]': finished 2024-09-17T01:27:03.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:03 smithi105 ceph-mon[24602]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T01:27:03.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:03 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/53756042' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:03 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]: dispatch 2024-09-17T01:27:03.637 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:27:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:27:03.638 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:27:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:27:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:03] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-09-17T01:27:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]': finished 2024-09-17T01:27:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T01:27:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]': finished 2024-09-17T01:27:04.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1296425100"}]': finished 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:27:04.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: Checking dashboard <-> RGW credentials 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 767 B/s wr, 72 op/s 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]': finished 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]: dispatch 2024-09-17T01:27:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:27:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: Checking dashboard <-> RGW credentials 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 767 B/s wr, 72 op/s 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]': finished 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]: dispatch 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: Checking dashboard <-> RGW credentials 2024-09-17T01:27:05.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 767 B/s wr, 72 op/s 2024-09-17T01:27:05.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/331499648' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/547777509"}]': finished 2024-09-17T01:27:05.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T01:27:05.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]: dispatch 2024-09-17T01:27:05.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:27:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:06 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]': finished 2024-09-17T01:27:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:06 smithi105 ceph-mon[24602]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T01:27:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:06 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/386370031' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:06 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]': finished 2024-09-17T01:27:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[23696]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T01:27:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/386370031' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1305184664' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1568007122"}]': finished 2024-09-17T01:27:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[28146]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T01:27:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/386370031' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:06 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]: dispatch 2024-09-17T01:27:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:07 smithi105 ceph-mon[24602]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:07 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]': finished 2024-09-17T01:27:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:07 smithi105 ceph-mon[24602]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T01:27:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:07 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]: dispatch 2024-09-17T01:27:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[23696]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]': finished 2024-09-17T01:27:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[23696]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T01:27:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]: dispatch 2024-09-17T01:27:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[28146]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:07.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1138093002"}]': finished 2024-09-17T01:27:07.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[28146]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T01:27:07.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:07 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]: dispatch 2024-09-17T01:27:08.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:08 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]': finished 2024-09-17T01:27:08.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:08 smithi105 ceph-mon[24602]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T01:27:08.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:08 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1655319789' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:08.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:08 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]': finished 2024-09-17T01:27:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[23696]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T01:27:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1655319789' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1935081616' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/547777509"}]': finished 2024-09-17T01:27:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[28146]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T01:27:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1655319789' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:08 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]: dispatch 2024-09-17T01:27:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:09 smithi105 ceph-mon[24602]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:09 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]': finished 2024-09-17T01:27:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:09 smithi105 ceph-mon[24602]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T01:27:09.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:09 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3152400584' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:09.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:09 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[23696]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]': finished 2024-09-17T01:27:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[23696]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T01:27:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3152400584' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[28146]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-09-17T01:27:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/742413165"}]': finished 2024-09-17T01:27:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[28146]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T01:27:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3152400584' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:09 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]: dispatch 2024-09-17T01:27:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:10 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]': finished 2024-09-17T01:27:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:10 smithi105 ceph-mon[24602]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T01:27:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:10 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]: dispatch 2024-09-17T01:27:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]': finished 2024-09-17T01:27:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[23696]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T01:27:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]: dispatch 2024-09-17T01:27:10.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/662934856"}]': finished 2024-09-17T01:27:10.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[28146]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T01:27:10.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:10 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]: dispatch 2024-09-17T01:27:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:27:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]': finished 2024-09-17T01:27:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T01:27:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2868114858' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:11 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]': finished 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2868114858' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/33162681' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2620727570"}]': finished 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2868114858' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]: dispatch 2024-09-17T01:27:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:11 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:12.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:12 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]': finished 2024-09-17T01:27:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:12 smithi105 ceph-mon[24602]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T01:27:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:12 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3633204704' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:12 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]': finished 2024-09-17T01:27:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[23696]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T01:27:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3633204704' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1138093002"}]': finished 2024-09-17T01:27:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[28146]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T01:27:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3633204704' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:12 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]: dispatch 2024-09-17T01:27:13.112 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:13 smithi105 ceph-mon[24602]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:13 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]': finished 2024-09-17T01:27:13.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:13 smithi105 ceph-mon[24602]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T01:27:13.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:13 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]: dispatch 2024-09-17T01:27:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[23696]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]': finished 2024-09-17T01:27:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[23696]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T01:27:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]: dispatch 2024-09-17T01:27:13.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[28146]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3873552392"}]': finished 2024-09-17T01:27:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[28146]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T01:27:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]: dispatch 2024-09-17T01:27:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:13] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-09-17T01:27:13.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=error ts=2024-09-17T01:27:13.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:27:13.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:27:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:27:13.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:27:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:27:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]': finished 2024-09-17T01:27:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:14 smithi105 ceph-mon[24602]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T01:27:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]: dispatch 2024-09-17T01:27:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]': finished 2024-09-17T01:27:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[28146]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T01:27:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]: dispatch 2024-09-17T01:27:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2119430453' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1710452097"}]': finished 2024-09-17T01:27:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[23696]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T01:27:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]: dispatch 2024-09-17T01:27:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:15 smithi105 ceph-mon[24602]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:15 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]': finished 2024-09-17T01:27:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:15 smithi105 ceph-mon[24602]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T01:27:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:15 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]: dispatch 2024-09-17T01:27:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[28146]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]': finished 2024-09-17T01:27:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[28146]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T01:27:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]: dispatch 2024-09-17T01:27:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[23696]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/971372728' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/597110760"}]': finished 2024-09-17T01:27:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[23696]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T01:27:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:15 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]: dispatch 2024-09-17T01:27:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:16 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]': finished 2024-09-17T01:27:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:16 smithi105 ceph-mon[24602]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T01:27:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:16 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]: dispatch 2024-09-17T01:27:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]': finished 2024-09-17T01:27:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[23696]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T01:27:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]: dispatch 2024-09-17T01:27:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1036012881' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1807582732"}]': finished 2024-09-17T01:27:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[28146]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T01:27:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:16 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]: dispatch 2024-09-17T01:27:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:17 smithi105 ceph-mon[24602]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:17 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]': finished 2024-09-17T01:27:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:17 smithi105 ceph-mon[24602]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T01:27:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:17 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]: dispatch 2024-09-17T01:27:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[23696]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]': finished 2024-09-17T01:27:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[23696]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T01:27:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]: dispatch 2024-09-17T01:27:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[28146]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1537767071' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1436016564"}]': finished 2024-09-17T01:27:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[28146]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T01:27:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:17 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]: dispatch 2024-09-17T01:27:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:18 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]': finished 2024-09-17T01:27:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:18 smithi081 ceph-mon[23696]: osdmap e95: 8 total, 8 up, 8 in 2024-09-17T01:27:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:18 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]': finished 2024-09-17T01:27:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:18 smithi081 ceph-mon[28146]: osdmap e95: 8 total, 8 up, 8 in 2024-09-17T01:27:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:18 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/595432004' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/597110760"}]': finished 2024-09-17T01:27:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:18 smithi105 ceph-mon[24602]: osdmap e95: 8 total, 8 up, 8 in 2024-09-17T01:27:19.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:19 smithi081 ceph-mon[23696]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:19.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:19 smithi081 ceph-mon[28146]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:19 smithi105 ceph-mon[24602]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:21 smithi081 ceph-mon[23696]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-09-17T01:27:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:21 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:21.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:21 smithi081 ceph-mon[28146]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-09-17T01:27:21.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:21 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:21 smithi105 ceph-mon[24602]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 76 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-09-17T01:27:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:21 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:23.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:23 smithi081 ceph-mon[23696]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:27:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:23 smithi081 ceph-mon[28146]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:27:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:23 smithi105 ceph-mon[24602]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:27:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:23] "GET /metrics HTTP/1.1" 200 208133 "" "Prometheus/2.33.4" 2024-09-17T01:27:23.930 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:27:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=error ts=2024-09-17T01:27:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:27:25.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:25 smithi081 ceph-mon[23696]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:27:25.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:25 smithi081 ceph-mon[28146]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:27:25.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:25 smithi105 ceph-mon[24602]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:27:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:27 smithi081 ceph-mon[23696]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:27:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:27 smithi081 ceph-mon[28146]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:27:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:27 smithi105 ceph-mon[24602]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:27:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:29 smithi081 ceph-mon[23696]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-09-17T01:27:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:29 smithi081 ceph-mon[28146]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-09-17T01:27:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:29 smithi105 ceph-mon[24602]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-09-17T01:27:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:31 smithi081 ceph-mon[23696]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:31 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:31 smithi081 ceph-mon[28146]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:31.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:31 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:31 smithi105 ceph-mon[24602]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:31 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:33.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:33 smithi081 ceph-mon[23696]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:33.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:33 smithi081 ceph-mon[28146]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:33 smithi105 ceph-mon[24602]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:33] "GET /metrics HTTP/1.1" 200 208088 "" "Prometheus/2.33.4" 2024-09-17T01:27:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:35 smithi081 ceph-mon[23696]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:35.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:35 smithi081 ceph-mon[28146]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:35 smithi105 ceph-mon[24602]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:37 smithi081 ceph-mon[23696]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:37.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:37 smithi081 ceph-mon[28146]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:37 smithi105 ceph-mon[24602]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:39 smithi105 ceph-mon[24602]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:39 smithi081 ceph-mon[23696]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:39 smithi081 ceph-mon[28146]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:41 smithi105 ceph-mon[24602]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:41 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:41 smithi081 ceph-mon[23696]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:41 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:41 smithi081 ceph-mon[28146]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:41 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:43.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:43 smithi105 ceph-mon[24602]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:43.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:43 smithi081 ceph-mon[23696]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:43.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:43 smithi081 ceph-mon[28146]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:43] "GET /metrics HTTP/1.1" 200 208088 "" "Prometheus/2.33.4" 2024-09-17T01:27:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:45 smithi105 ceph-mon[24602]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:45 smithi081 ceph-mon[23696]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:45 smithi081 ceph-mon[28146]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:47 smithi105 ceph-mon[24602]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:47 smithi081 ceph-mon[23696]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:47 smithi081 ceph-mon[28146]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:49 smithi105 ceph-mon[24602]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:49 smithi081 ceph-mon[23696]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:49 smithi081 ceph-mon[28146]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:51 smithi105 ceph-mon[24602]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:51 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:51 smithi081 ceph-mon[28146]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:51 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:51.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:51 smithi081 ceph-mon[23696]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:51.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:51 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:27:53.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:27:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:27:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:53 smithi105 ceph-mon[24602]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:53.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:53 smithi081 ceph-mon[23696]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:53.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:53 smithi081 ceph-mon[28146]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:27:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:27:53] "GET /metrics HTTP/1.1" 200 208040 "" "Prometheus/2.33.4" 2024-09-17T01:27:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:55 smithi105 ceph-mon[24602]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:55 smithi081 ceph-mon[23696]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:55 smithi081 ceph-mon[28146]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:57 smithi105 ceph-mon[24602]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:57 smithi081 ceph-mon[23696]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:57 smithi081 ceph-mon[28146]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:27:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:27:59 smithi105 ceph-mon[24602]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:27:59 smithi081 ceph-mon[23696]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:27:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:27:59 smithi081 ceph-mon[28146]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:00.596 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:28:00.597 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:28:00.597 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:28:00.597 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:28:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:28:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:28:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:01 smithi105 ceph-mon[24602]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:01 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:01 smithi081 ceph-mon[23696]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:01 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:01 smithi081 ceph-mon[28146]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:01 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:03.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:03 smithi105 ceph-mon[24602]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:03.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:03 smithi081 ceph-mon[23696]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:03.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:03 smithi081 ceph-mon[28146]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:03] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-09-17T01:28:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:28:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:28:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:28:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:28:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:28:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:28:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:28:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:28:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:28:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:05 smithi105 ceph-mon[24602]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:05 smithi081 ceph-mon[23696]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:05.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:05 smithi081 ceph-mon[28146]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:07 smithi105 ceph-mon[24602]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:07.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:07 smithi081 ceph-mon[23696]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:07.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:07 smithi081 ceph-mon[28146]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:09 smithi105 ceph-mon[24602]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:09 smithi081 ceph-mon[23696]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:09 smithi081 ceph-mon[28146]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:11 smithi105 ceph-mon[24602]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:11.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:11 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:11 smithi081 ceph-mon[23696]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:11 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:11 smithi081 ceph-mon[28146]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:11 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:13 smithi105 ceph-mon[24602]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:13.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:13 smithi081 ceph-mon[23696]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:13.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:13 smithi081 ceph-mon[28146]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:13] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-09-17T01:28:15.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:15 smithi105 ceph-mon[24602]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:15.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:15 smithi081 ceph-mon[28146]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:15.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:15 smithi081 ceph-mon[23696]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:17 smithi105 ceph-mon[24602]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:17.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:17 smithi081 ceph-mon[23696]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:17.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:17 smithi081 ceph-mon[28146]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:19 smithi105 ceph-mon[24602]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:19 smithi081 ceph-mon[23696]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:19 smithi081 ceph-mon[28146]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:21 smithi105 ceph-mon[24602]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:21 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:21 smithi081 ceph-mon[23696]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:21 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:21 smithi081 ceph-mon[28146]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:21 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:23 smithi105 ceph-mon[24602]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:23.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:23 smithi081 ceph-mon[23696]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:23.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:23 smithi081 ceph-mon[28146]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:23] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-09-17T01:28:25.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:25 smithi105 ceph-mon[24602]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:25.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:25 smithi081 ceph-mon[28146]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:25 smithi081 ceph-mon[23696]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:27 smithi105 ceph-mon[24602]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:27 smithi081 ceph-mon[23696]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:27 smithi081 ceph-mon[28146]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:29 smithi105 ceph-mon[24602]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:29 smithi081 ceph-mon[23696]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:29 smithi081 ceph-mon[28146]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:31 smithi105 ceph-mon[24602]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:31.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:31 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:31 smithi081 ceph-mon[23696]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:31 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:31 smithi081 ceph-mon[28146]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:31.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:31 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:33.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:33 smithi105 ceph-mon[24602]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:33.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:33 smithi081 ceph-mon[23696]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:33.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:33 smithi081 ceph-mon[28146]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:33] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-09-17T01:28:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:35 smithi105 ceph-mon[24602]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:35.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:35 smithi081 ceph-mon[23696]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:35.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:35 smithi081 ceph-mon[28146]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:37 smithi105 ceph-mon[24602]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:37 smithi081 ceph-mon[23696]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:37 smithi081 ceph-mon[28146]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:39 smithi105 ceph-mon[24602]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:39 smithi081 ceph-mon[23696]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:39 smithi081 ceph-mon[28146]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:41 smithi105 ceph-mon[24602]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:41 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:41 smithi081 ceph-mon[23696]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:41 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:41 smithi081 ceph-mon[28146]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:41 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:43.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:43 smithi105 ceph-mon[24602]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:43.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:43 smithi081 ceph-mon[23696]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:43.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:43 smithi081 ceph-mon[28146]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:43] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-09-17T01:28:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:45 smithi105 ceph-mon[24602]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:45 smithi081 ceph-mon[23696]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:45 smithi081 ceph-mon[28146]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:47 smithi105 ceph-mon[24602]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:47 smithi081 ceph-mon[23696]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:47 smithi081 ceph-mon[28146]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:49 smithi105 ceph-mon[24602]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:49 smithi081 ceph-mon[23696]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:49 smithi081 ceph-mon[28146]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:51 smithi105 ceph-mon[24602]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:51 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:51 smithi081 ceph-mon[23696]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:51 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:51 smithi081 ceph-mon[28146]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:28:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:51 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:28:53.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:28:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:28:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:53 smithi105 ceph-mon[24602]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:53.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:53 smithi081 ceph-mon[23696]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:53.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:53 smithi081 ceph-mon[28146]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:28:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:28:53] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-09-17T01:28:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:55 smithi105 ceph-mon[24602]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:55 smithi081 ceph-mon[23696]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:55 smithi081 ceph-mon[28146]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:57 smithi105 ceph-mon[24602]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:57 smithi081 ceph-mon[23696]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:57 smithi081 ceph-mon[28146]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:28:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:28:59 smithi105 ceph-mon[24602]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:28:59 smithi081 ceph-mon[23696]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:28:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:28:59 smithi081 ceph-mon[28146]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:29:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:29:00.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:29:00.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:29:00.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:29:00.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:29:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:01 smithi105 ceph-mon[24602]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:01 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:01 smithi081 ceph-mon[23696]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:01 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:01 smithi081 ceph-mon[28146]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:01 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:03.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:03 smithi105 ceph-mon[24602]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:03.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:03 smithi081 ceph-mon[23696]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:03.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:03 smithi081 ceph-mon[28146]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:03] "GET /metrics HTTP/1.1" 200 208028 "" "Prometheus/2.33.4" 2024-09-17T01:29:04.485 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:29:04.485 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:29:04.487 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:29:04.487 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:29:04.487 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:29:04.487 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:29:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:05 smithi105 ceph-mon[24602]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:29:05.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:05 smithi081 ceph-mon[23696]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:05.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:29:05.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:05 smithi081 ceph-mon[28146]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:05.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:29:07.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:07 smithi105 ceph-mon[24602]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:07.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:07 smithi081 ceph-mon[23696]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:07.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:07 smithi081 ceph-mon[28146]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:09.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:09 smithi105 ceph-mon[24602]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:09 smithi081 ceph-mon[23696]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:09 smithi081 ceph-mon[28146]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:11.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:11 smithi105 ceph-mon[24602]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:11.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:11 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:11.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:11 smithi081 ceph-mon[23696]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:11.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:11 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:11.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:11 smithi081 ceph-mon[28146]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:11.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:11 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:13.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:13 smithi105 ceph-mon[24602]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:13 smithi081 ceph-mon[23696]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:13] "GET /metrics HTTP/1.1" 200 208028 "" "Prometheus/2.33.4" 2024-09-17T01:29:13.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:13 smithi081 ceph-mon[28146]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:15.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:15 smithi105 ceph-mon[24602]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:15 smithi081 ceph-mon[23696]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:15.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:15 smithi081 ceph-mon[28146]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:17.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:17 smithi105 ceph-mon[24602]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:17.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:17 smithi081 ceph-mon[23696]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:17.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:17 smithi081 ceph-mon[28146]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:19.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:19 smithi105 ceph-mon[24602]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:19.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:19 smithi081 ceph-mon[23696]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:19.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:19 smithi081 ceph-mon[28146]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:21 smithi105 ceph-mon[24602]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:21.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:21 smithi081 ceph-mon[23696]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:21.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:21 smithi081 ceph-mon[28146]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:22.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:22 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:22.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:22 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:22.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:22 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:23 smithi105 ceph-mon[24602]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:23 smithi081 ceph-mon[23696]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:23.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:23 smithi081 ceph-mon[28146]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:23.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:23] "GET /metrics HTTP/1.1" 200 208023 "" "Prometheus/2.33.4" 2024-09-17T01:29:25.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:25 smithi105 ceph-mon[24602]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:25.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:25 smithi081 ceph-mon[23696]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:25.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:25 smithi081 ceph-mon[28146]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:27.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:27 smithi105 ceph-mon[24602]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:27.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:27 smithi081 ceph-mon[23696]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:27.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:27 smithi081 ceph-mon[28146]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:29.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:29 smithi105 ceph-mon[24602]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:29.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:29 smithi081 ceph-mon[23696]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:29.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:29 smithi081 ceph-mon[28146]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:31.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:31 smithi105 ceph-mon[24602]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:31.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:31 smithi081 ceph-mon[23696]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:31.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:31 smithi081 ceph-mon[28146]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:32.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:32 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:32.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:32.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:32 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:32 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:33.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:33 smithi105 ceph-mon[24602]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:33.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:33 smithi081 ceph-mon[23696]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:33 smithi081 ceph-mon[28146]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:33.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:33] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-09-17T01:29:35.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:35 smithi105 ceph-mon[24602]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:35.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:35 smithi081 ceph-mon[23696]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:35.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:35 smithi081 ceph-mon[28146]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:37.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:37 smithi105 ceph-mon[24602]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:37 smithi081 ceph-mon[23696]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:37.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:37 smithi081 ceph-mon[28146]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:39 smithi105 ceph-mon[24602]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:39.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:39 smithi081 ceph-mon[23696]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:39.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:39 smithi081 ceph-mon[28146]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:41.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:41 smithi105 ceph-mon[24602]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:41.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:41 smithi081 ceph-mon[23696]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:41.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:41 smithi081 ceph-mon[28146]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:42 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:42.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:42 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:42.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:42 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:43.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:43 smithi105 ceph-mon[24602]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:43.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:43 smithi081 ceph-mon[23696]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:43.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:43 smithi081 ceph-mon[28146]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:43] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-09-17T01:29:45.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:45 smithi105 ceph-mon[24602]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:45.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:45 smithi081 ceph-mon[23696]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:45.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:45 smithi081 ceph-mon[28146]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:47.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:47 smithi105 ceph-mon[24602]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:47.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:47 smithi081 ceph-mon[23696]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:47.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:47 smithi081 ceph-mon[28146]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:49 smithi105 ceph-mon[24602]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:49 smithi081 ceph-mon[23696]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:49.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:49 smithi081 ceph-mon[28146]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:51.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:51 smithi105 ceph-mon[24602]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:51.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:51 smithi081 ceph-mon[23696]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:51.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:51 smithi081 ceph-mon[28146]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:52 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:52.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:29:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:29:52.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:52 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:52.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:52 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:29:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:53 smithi105 ceph-mon[24602]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:53 smithi081 ceph-mon[23696]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:29:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:29:53] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-09-17T01:29:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:53 smithi081 ceph-mon[28146]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:55.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:55 smithi105 ceph-mon[24602]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:55.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:55 smithi081 ceph-mon[23696]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:55.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:55 smithi081 ceph-mon[28146]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:57 smithi105 ceph-mon[24602]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:57.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:57 smithi081 ceph-mon[23696]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:57.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:57 smithi081 ceph-mon[28146]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:29:58.370 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T01:29:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:29:59 smithi081 ceph-mon[23696]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:29:59 smithi081 ceph-mon[28146]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:29:59.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:29:59 smithi105 ceph-mon[24602]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:00.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:30:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:30:00.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:00 smithi105 ceph-mon[24602]: overall HEALTH_OK 2024-09-17T01:30:00.918 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:30:00.997 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:30:00.998 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[23696]: overall HEALTH_OK 2024-09-17T01:30:00.998 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:30:00.998 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:30:00.998 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:00 smithi081 ceph-mon[28146]: overall HEALTH_OK 2024-09-17T01:30:00.999 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T01:30:01.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:01 smithi105 ceph-mon[24602]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:01.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:01 smithi081 ceph-mon[23696]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:01.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:01 smithi081 ceph-mon[28146]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:02.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:02 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:02.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:02.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:02 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:02.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:02 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:03.598 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T01:30:03.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:03 smithi105 ceph-mon[24602]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:03.897 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:03 smithi081 ceph-mon[23696]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:03.898 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:03 smithi081 ceph-mon[28146]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:03.898 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:03] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-09-17T01:30:04.797 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:04.797 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:04.898 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:04.898 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:04.898 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:04.899 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:05.864 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:05 smithi081 ceph-mon[23696]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:05.864 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:05 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:05.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:05 smithi081 ceph-mon[28146]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:05.864 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:05 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:05 smithi105 ceph-mon[24602]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:05 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:06.213 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:30:07.701 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:07 smithi081 ceph-mon[23696]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:07.701 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:07 smithi081 ceph-mon[28146]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:07.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:07 smithi105 ceph-mon[24602]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:08.098 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:30:08.098 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c 1d46348ac778 2024-09-17T01:30:08.098 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (3m) 3m ago 3m 47.7M - 8.3.5 dad864ee21e9 d2a64aee5705 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (3m) 3m ago 3m 71.9M - 3.5 e1d6a67b021e d964d8155594 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e faaaba0463c2 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e a454a71b7b66 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (8m) 3m ago 8m 41.2M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (8m) 3m ago 8m 37.8M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (4m) 3m ago 4m 17.1M - 1dbe0e931976 8b4df6dc354d 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (4m) 3m ago 4m 18.2M - 1dbe0e931976 8fa0015a15e5 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (8m) 3m ago 8m 48.3M 4016M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (7m) 3m ago 7m 48.5M 4016M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (7m) 3m ago 7m 43.8M 4016M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (7m) 3m ago 7m 46.2M 4016M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:30:08.099 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (6m) 3m ago 6m 46.0M 4272M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (6m) 3m ago 6m 43.8M 4272M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (5m) 3m ago 5m 44.4M 4272M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (5m) 3m ago 5m 44.5M 4272M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (3m) 3m ago 4m 43.8M - 514e6a882f6e 2b7eda538c73 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (3m) 3m ago 3m 71.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:30:08.100 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (3m) 3m ago 3m 71.4M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:30:08.777 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:30:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:09 smithi105 ceph-mon[24602]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:09 smithi105 ceph-mon[24602]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:09 smithi081 ceph-mon[23696]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:09 smithi081 ceph-mon[23696]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:09 smithi081 ceph-mon[28146]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:09 smithi081 ceph-mon[28146]: from='client.24859 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:10.761 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:30:10.761 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:30:10.762 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:30:10.763 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-09-17T01:30:10.763 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:30:10.763 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:30:11.476 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph -s' 2024-09-17T01:30:11.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:11 smithi105 ceph-mon[24602]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:11.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2311515186' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:30:11.890 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:11 smithi081 ceph-mon[23696]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:11.890 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2311515186' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:30:11.890 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:11 smithi081 ceph-mon[28146]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:11.890 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2311515186' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:30:12.893 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:12 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:12.893 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:12 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:13.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:12 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:13.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:13.336 INFO:teuthology.orchestra.run.smithi081.stdout: cluster: 2024-09-17T01:30:13.336 INFO:teuthology.orchestra.run.smithi081.stdout: id: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:30:13.336 INFO:teuthology.orchestra.run.smithi081.stdout: health: HEALTH_OK 2024-09-17T01:30:13.336 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:30:13.336 INFO:teuthology.orchestra.run.smithi081.stdout: services: 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: mgr: y(active, since 5m), standbys: x 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: data: 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: pools: 6 pools, 161 pgs 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: objects: 241 objects, 585 KiB 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: pgs: 161 active+clean 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: io: 2024-09-17T01:30:13.337 INFO:teuthology.orchestra.run.smithi081.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-09-17T01:30:13.338 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:30:13.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:13 smithi081 ceph-mon[28146]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:13.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:13 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/738792074' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:30:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:13 smithi081 ceph-mon[23696]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:13.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:13 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/738792074' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:30:13.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:13] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-09-17T01:30:13.986 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ls' 2024-09-17T01:30:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:13 smithi105 ceph-mon[24602]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:14.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:13 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/738792074' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:30:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:15 smithi081 ceph-mon[23696]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:15.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:15 smithi081 ceph-mon[28146]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:15.932 INFO:teuthology.orchestra.run.smithi081.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T01:30:15.932 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi081=a;count:1 2024-09-17T01:30:15.932 INFO:teuthology.orchestra.run.smithi081.stdout:grafana ?:3000 1/1 3m ago 5m smithi105=a;count:1 2024-09-17T01:30:15.932 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:mgr 2/2 3m ago 8m smithi081=y;smithi105=x;count:2 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:mon 3/3 3m ago 9m smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi081=a;smithi105=b;count:2 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:osd 8 3m ago - 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus ?:9095 1/1 3m ago 5m smithi105=a;count:1 2024-09-17T01:30:15.933 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-09-17T01:30:16.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:15 smithi105 ceph-mon[24602]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:16.619 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T01:30:17.643 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:17 smithi081 ceph-mon[23696]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:17.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:17 smithi081 ceph-mon[23696]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:17.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:17 smithi081 ceph-mon[28146]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:17.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:17 smithi081 ceph-mon[28146]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:18.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:17 smithi105 ceph-mon[24602]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:18.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:17 smithi105 ceph-mon[24602]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:19.078 INFO:teuthology.orchestra.run.smithi081.stdout:Scheduled to redeploy mgr.x on host 'smithi105' 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/541664917' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:19.800 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/541664917' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:30:19.801 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:19 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:19.823 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps --refresh' 2024-09-17T01:30:20.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/541664917' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:30:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:19 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:30:20.723 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[28146]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:20.723 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[28146]: Schedule redeploy daemon mgr.x 2024-09-17T01:30:20.723 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[28146]: Deploying daemon mgr.x on smithi105 2024-09-17T01:30:20.724 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[23696]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:20.724 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[23696]: Schedule redeploy daemon mgr.x 2024-09-17T01:30:20.724 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:20 smithi081 ceph-mon[23696]: Deploying daemon mgr.x on smithi105 2024-09-17T01:30:21.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:20 smithi105 ceph-mon[24602]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:21.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:20 smithi105 ceph-mon[24602]: Schedule redeploy daemon mgr.x 2024-09-17T01:30:21.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:20 smithi105 ceph-mon[24602]: Deploying daemon mgr.x on smithi105 2024-09-17T01:30:21.871 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:21 smithi081 ceph-mon[23696]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c 1d46348ac778 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (4m) 3m ago 4m 47.7M - 8.3.5 dad864ee21e9 d2a64aee5705 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (3m) 3m ago 3m 71.9M - 3.5 e1d6a67b021e d964d8155594 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e faaaba0463c2 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e a454a71b7b66 2024-09-17T01:30:21.872 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (8m) 3m ago 8m 41.2M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (9m) 3m ago 9m 37.8M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (5m) 3m ago 5m 17.1M - 1dbe0e931976 8b4df6dc354d 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (4m) 3m ago 4m 18.2M - 1dbe0e931976 8fa0015a15e5 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (8m) 3m ago 8m 48.3M 4016M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (8m) 3m ago 8m 48.5M 4016M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (7m) 3m ago 7m 43.8M 4016M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (7m) 3m ago 7m 46.2M 4016M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (6m) 3m ago 6m 46.0M 4272M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (6m) 3m ago 6m 43.8M 4272M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (6m) 3m ago 6m 44.4M 4272M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (5m) 3m ago 5m 44.5M 4272M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:30:21.873 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (3m) 3m ago 5m 43.8M - 514e6a882f6e 2b7eda538c73 2024-09-17T01:30:21.874 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (3m) 3m ago 3m 71.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:30:21.874 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (3m) 3m ago 3m 71.4M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:30:21.874 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:21 smithi081 ceph-mon[28146]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:22.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:21 smithi105 ceph-mon[24602]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:22.639 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:30:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:22 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:22 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:22 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:23.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:23 smithi081 ceph-mon[28146]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:23.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:23 smithi081 ceph-mon[28146]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:23.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:23 smithi081 ceph-mon[23696]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:23.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:23 smithi081 ceph-mon[23696]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:23.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:23] "GET /metrics HTTP/1.1" 200 207963 "" "Prometheus/2.33.4" 2024-09-17T01:30:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:23 smithi105 ceph-mon[24602]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:23 smithi105 ceph-mon[24602]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:30:25.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:25 smithi081 ceph-mon[23696]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:25.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:25 smithi081 ceph-mon[28146]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:25.936 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:25 smithi105 ceph-mon[24602]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:27.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:27 smithi081 ceph-mon[23696]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:27.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:27 smithi081 ceph-mon[28146]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:28.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:27 smithi105 ceph-mon[24602]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:29.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:29 smithi081 ceph-mon[23696]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:29.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:29 smithi081 ceph-mon[28146]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:29 smithi105 ceph-mon[24602]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:31.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:31 smithi081 ceph-mon[23696]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:31.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:31 smithi081 ceph-mon[28146]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:31 smithi105 ceph-mon[24602]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:32.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:32 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:32 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:33.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:32 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:33.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:33.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:33 smithi081 ceph-mon[23696]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:33.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:33 smithi081 ceph-mon[28146]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:33.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:33] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-09-17T01:30:34.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:33 smithi105 ceph-mon[24602]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:35.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:35 smithi081 ceph-mon[23696]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:35.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:35 smithi081 ceph-mon[28146]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:36.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:35 smithi105 ceph-mon[24602]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:37 smithi081 ceph-mon[23696]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:37.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:37 smithi081 ceph-mon[28146]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:37 smithi105 ceph-mon[24602]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:39.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:39 smithi081 ceph-mon[23696]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:39.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:39 smithi081 ceph-mon[28146]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:40.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:39 smithi105 ceph-mon[24602]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:42.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:41 smithi105 ceph-mon[24602]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:42.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:41 smithi081 ceph-mon[23696]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:42.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:41 smithi081 ceph-mon[28146]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:43.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:42 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:43.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:43.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:42 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:43.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:42 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:43.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:43 smithi081 ceph-mon[23696]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:43.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:43 smithi081 ceph-mon[28146]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:43.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:43] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-09-17T01:30:44.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:43 smithi105 ceph-mon[24602]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:44 smithi105 ceph-mon[24602]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:45.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:44 smithi081 ceph-mon[23696]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:44 smithi081 ceph-mon[28146]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:47 smithi105 ceph-mon[24602]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:47 smithi081 ceph-mon[23696]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:47.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:47 smithi081 ceph-mon[28146]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:49 smithi105 ceph-mon[24602]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:49.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:49 smithi081 ceph-mon[23696]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:49 smithi081 ceph-mon[28146]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:51 smithi105 ceph-mon[24602]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:51 smithi081 ceph-mon[23696]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:51.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:51 smithi081 ceph-mon[28146]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:52 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:52.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:52 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:52.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:52 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:30:53.094 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:30:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:30:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:53 smithi105 ceph-mon[24602]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:53.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:53 smithi081 ceph-mon[23696]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:53.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:53 smithi081 ceph-mon[28146]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:30:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:30:53] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-09-17T01:30:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:55 smithi105 ceph-mon[24602]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:55 smithi081 ceph-mon[23696]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:55.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:55 smithi081 ceph-mon[28146]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:57 smithi105 ceph-mon[24602]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:57 smithi081 ceph-mon[23696]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:57 smithi081 ceph-mon[28146]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:30:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:30:59 smithi105 ceph-mon[24602]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:30:59 smithi081 ceph-mon[23696]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:30:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:30:59 smithi081 ceph-mon[28146]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:31:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:31:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:31:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:31:00.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:31:00.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:31:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:01 smithi105 ceph-mon[24602]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:01.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:01 smithi081 ceph-mon[28146]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:01.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:01 smithi081 ceph-mon[23696]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:02 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:02 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:02.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:02 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:03.119 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:03 smithi105 ceph-mon[24602]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:03.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:03 smithi081 ceph-mon[28146]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:03.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:03 smithi081 ceph-mon[23696]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:03] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-09-17T01:31:05.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:05 smithi105 ceph-mon[24602]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:05 smithi081 ceph-mon[23696]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:05 smithi081 ceph-mon[28146]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:07 smithi105 ceph-mon[24602]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:07 smithi081 ceph-mon[23696]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:07 smithi081 ceph-mon[28146]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:09 smithi105 ceph-mon[24602]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:09 smithi081 ceph-mon[23696]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:09.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:09 smithi081 ceph-mon[28146]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:11 smithi105 ceph-mon[24602]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:11 smithi081 ceph-mon[23696]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:11.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:11 smithi081 ceph-mon[28146]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:12 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:12.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:12 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:12 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:13 smithi081 ceph-mon[23696]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:13.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:13 smithi081 ceph-mon[28146]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:13 smithi105 ceph-mon[24602]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:13] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-09-17T01:31:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:15 smithi081 ceph-mon[23696]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:15 smithi081 ceph-mon[28146]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:15.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:15 smithi105 ceph-mon[24602]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:17 smithi081 ceph-mon[23696]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:17.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:17 smithi081 ceph-mon[28146]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:17 smithi105 ceph-mon[24602]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:19.405 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:19 smithi081 ceph-mon[23696]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:19.406 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:19 smithi081 ceph-mon[28146]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:19 smithi105 ceph-mon[24602]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:21.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:21 smithi081 ceph-mon[23696]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:21.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:21 smithi081 ceph-mon[28146]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:21 smithi105 ceph-mon[24602]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:22.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:22 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:22.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:22 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:22 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:23.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:23 smithi081 ceph-mon[23696]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:23.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:23 smithi081 ceph-mon[28146]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:23 smithi105 ceph-mon[24602]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:23] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-09-17T01:31:25.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:25 smithi081 ceph-mon[23696]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:25.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:25 smithi081 ceph-mon[28146]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:25.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:25 smithi105 ceph-mon[24602]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:27 smithi081 ceph-mon[23696]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:27 smithi081 ceph-mon[28146]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:27 smithi105 ceph-mon[24602]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:29 smithi081 ceph-mon[28146]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:29 smithi081 ceph-mon[23696]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:29 smithi105 ceph-mon[24602]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:31 smithi081 ceph-mon[23696]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:31 smithi081 ceph-mon[28146]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:31 smithi105 ceph-mon[24602]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:32 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:32 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:32 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:33.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:33 smithi105 ceph-mon[24602]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:33.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:33 smithi081 ceph-mon[23696]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:33.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:33 smithi081 ceph-mon[28146]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:33] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-17T01:31:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:35 smithi105 ceph-mon[24602]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:35.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:35 smithi081 ceph-mon[28146]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:35.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:35 smithi081 ceph-mon[23696]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:37 smithi105 ceph-mon[24602]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:37 smithi081 ceph-mon[28146]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:37 smithi081 ceph-mon[23696]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:39 smithi105 ceph-mon[24602]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:39 smithi081 ceph-mon[28146]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:39 smithi081 ceph-mon[23696]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:41 smithi105 ceph-mon[24602]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:41 smithi081 ceph-mon[28146]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:41 smithi081 ceph-mon[23696]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:42 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:42 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:42 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:43.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:43 smithi105 ceph-mon[24602]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:43.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:43 smithi081 ceph-mon[23696]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:43.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:43 smithi081 ceph-mon[28146]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:43] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-17T01:31:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:45 smithi105 ceph-mon[24602]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:45 smithi081 ceph-mon[23696]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:45 smithi081 ceph-mon[28146]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:47.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:47 smithi081 ceph-mon[23696]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:47.482 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:47 smithi081 ceph-mon[28146]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:47 smithi105 ceph-mon[24602]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:49 smithi105 ceph-mon[24602]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:49 smithi081 ceph-mon[23696]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:49 smithi081 ceph-mon[28146]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:51 smithi105 ceph-mon[24602]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:51 smithi081 ceph-mon[23696]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:51 smithi081 ceph-mon[28146]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:52 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:52 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:52.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:52 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:31:53.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:31:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:31:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:53 smithi105 ceph-mon[24602]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:53.675 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:53 smithi081 ceph-mon[28146]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:53.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:53 smithi081 ceph-mon[23696]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:31:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:31:53] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-17T01:31:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:55 smithi105 ceph-mon[24602]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:55 smithi081 ceph-mon[23696]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:55 smithi081 ceph-mon[28146]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:57 smithi105 ceph-mon[24602]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:57 smithi081 ceph-mon[23696]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:57 smithi081 ceph-mon[28146]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:31:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:31:59 smithi105 ceph-mon[24602]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:31:59 smithi081 ceph-mon[23696]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:31:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:31:59 smithi081 ceph-mon[28146]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:32:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:32:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:32:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:32:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:32:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:32:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:01 smithi105 ceph-mon[24602]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:01 smithi081 ceph-mon[23696]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:01 smithi081 ceph-mon[28146]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:02 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:02 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:02 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:03.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:03 smithi105 ceph-mon[24602]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:03.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:03 smithi081 ceph-mon[23696]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:03.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:03 smithi081 ceph-mon[28146]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:03] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-17T01:32:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:05 smithi105 ceph-mon[24602]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:05 smithi081 ceph-mon[23696]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:05.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:05 smithi081 ceph-mon[28146]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:07 smithi105 ceph-mon[24602]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:07.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:07 smithi081 ceph-mon[23696]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:07.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:07 smithi081 ceph-mon[28146]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:09 smithi105 ceph-mon[24602]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:09 smithi081 ceph-mon[23696]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:09 smithi081 ceph-mon[28146]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:11 smithi105 ceph-mon[24602]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:11 smithi081 ceph-mon[23696]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:11 smithi081 ceph-mon[28146]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:12 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:12 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:12 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:13 smithi105 ceph-mon[24602]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:13.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:13 smithi081 ceph-mon[23696]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:13.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:13 smithi081 ceph-mon[28146]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:13] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-09-17T01:32:15.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:15 smithi105 ceph-mon[24602]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:15.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:15 smithi081 ceph-mon[23696]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:15.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:15 smithi081 ceph-mon[28146]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:17 smithi105 ceph-mon[24602]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:17.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:17 smithi081 ceph-mon[23696]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:17.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:17 smithi081 ceph-mon[28146]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:19 smithi105 ceph-mon[24602]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:19 smithi081 ceph-mon[23696]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:19 smithi081 ceph-mon[28146]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:21 smithi105 ceph-mon[24602]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:21 smithi081 ceph-mon[23696]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:21 smithi081 ceph-mon[28146]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:22 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:22 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:22 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:23.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:23 smithi105 ceph-mon[24602]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:23.675 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:23 smithi081 ceph-mon[23696]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:23.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:23 smithi081 ceph-mon[28146]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:23] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-09-17T01:32:25.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:25 smithi105 ceph-mon[24602]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:25 smithi081 ceph-mon[23696]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:25.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:25 smithi081 ceph-mon[28146]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:27 smithi105 ceph-mon[24602]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:27 smithi081 ceph-mon[23696]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:27 smithi081 ceph-mon[28146]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:29 smithi105 ceph-mon[24602]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:29 smithi081 ceph-mon[23696]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:29 smithi081 ceph-mon[28146]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:31 smithi105 ceph-mon[24602]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:31 smithi081 ceph-mon[23696]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:31 smithi081 ceph-mon[28146]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:32 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:32 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:32 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:33.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:33 smithi105 ceph-mon[24602]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:33.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:33 smithi081 ceph-mon[23696]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:33.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:33 smithi081 ceph-mon[28146]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:33] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-09-17T01:32:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:35 smithi105 ceph-mon[24602]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:35.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:35 smithi081 ceph-mon[23696]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:35.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:35 smithi081 ceph-mon[28146]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:37 smithi105 ceph-mon[24602]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:37 smithi081 ceph-mon[23696]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:37 smithi081 ceph-mon[28146]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:39.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:39 smithi105 ceph-mon[24602]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:39.707 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:39 smithi081 ceph-mon[23696]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:39.707 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:39 smithi081 ceph-mon[28146]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:41 smithi105 ceph-mon[24602]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:41 smithi081 ceph-mon[23696]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:41 smithi081 ceph-mon[28146]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:42 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:42 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:42 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:43.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:43 smithi105 ceph-mon[24602]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:43.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:43 smithi081 ceph-mon[23696]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:43.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:43 smithi081 ceph-mon[28146]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:43] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-09-17T01:32:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:45 smithi105 ceph-mon[24602]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:45 smithi081 ceph-mon[23696]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:45 smithi081 ceph-mon[28146]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:47 smithi105 ceph-mon[24602]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:47 smithi081 ceph-mon[23696]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:47 smithi081 ceph-mon[28146]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:49 smithi105 ceph-mon[24602]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:49 smithi081 ceph-mon[23696]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:49 smithi081 ceph-mon[28146]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:51 smithi105 ceph-mon[24602]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:51 smithi081 ceph-mon[23696]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:51 smithi081 ceph-mon[28146]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:52 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:52.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:52 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:52 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:32:53.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:32:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[25771]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:32:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:53 smithi105 ceph-mon[24602]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:53.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:53 smithi081 ceph-mon[23696]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:53.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:53 smithi081 ceph-mon[28146]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:32:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:32:53] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-17T01:32:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:55 smithi105 ceph-mon[24602]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:55 smithi081 ceph-mon[23696]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:55 smithi081 ceph-mon[28146]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:57 smithi105 ceph-mon[24602]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:57 smithi081 ceph-mon[28146]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:57 smithi081 ceph-mon[23696]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:32:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:32:59 smithi105 ceph-mon[24602]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:32:59 smithi081 ceph-mon[23696]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:32:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:32:59 smithi081 ceph-mon[28146]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:00 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:00 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:00 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:01 smithi105 ceph-mon[24602]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:01 smithi081 ceph-mon[23696]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:01 smithi081 ceph-mon[28146]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:02.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 systemd[1]: Stopping Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:33:02.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 bash[44096]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-17T01:33:02.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 podman[44108]: 2024-09-17 01:33:02.272529016 +0000 UTC m=+0.145909085 container died faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, maintainer=Guillaume Abrioux , vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-09-17T01:33:02.655 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:02 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:02 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:02 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:02.915 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 podman[44108]: 2024-09-17 01:33:02.658213519 +0000 UTC m=+0.531593651 container cleanup faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, RELEASE=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T01:33:02.915 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 bash[44108]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x 2024-09-17T01:33:02.915 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 bash[44133]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr.x" found: no such container 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 podman[44120]: 2024-09-17 01:33:02.918107346 +0000 UTC m=+0.646596238 container remove faaaba0463c2066a5670e2ae0b03935e9374ac9907ba55bfe1a973841ece6e22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, version=8, GIT_CLEAN=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 systemd[1]: Stopped Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:02 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Consumed 18.123s CPU time. 2024-09-17T01:33:03.349 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 systemd[1]: Starting Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:33:03.621 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:03 smithi105 ceph-mon[24602]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:03.621 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 podman[44236]: 2024-09-17 01:33:03.254198521 +0000 UTC m=+0.015742102 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:33:03.622 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 podman[44236]: 2024-09-17 01:33:03.386490445 +0000 UTC m=+0.148034032 container create d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:33:03.622 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 podman[44236]: 2024-09-17 01:33:03.597241227 +0000 UTC m=+0.358784817 container init d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:33:03.622 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 podman[44236]: 2024-09-17 01:33:03.59997061 +0000 UTC m=+0.361514191 container start d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T01:33:03.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:03 smithi081 ceph-mon[23696]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:03.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:03 smithi081 ceph-mon[28146]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 ceph-mgr[44255]: -- 172.21.15.105:0/4091849214 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555d9935f0e0 con 0x555d9933d000 2024-09-17T01:33:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 bash[44236]: d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c 2024-09-17T01:33:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 systemd[1]: Started Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:33:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:03.730+0000 7f64f4362040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:33:03.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:03.793+0000 7f64f4362040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:33:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:03] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-17T01:33:04.335 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:03.953+0000 7f64f4362040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:33:04.335 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.026+0000 7f64f4362040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:33:04.335 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.183+0000 7f64f4362040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:33:04.335 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.336+0000 7f64f4362040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:33:04.925 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.669+0000 7f64f4362040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:33:04.925 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.737+0000 7f64f4362040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:04.927+0000 7f64f4362040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.926 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:04 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:04 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:06.275 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:05.900+0000 7f64f4362040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:33:06.275 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:05.983+0000 7f64f4362040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:33:07.036 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:06.772+0000 7f64f4362040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:33:07.036 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:06.891+0000 7f64f4362040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:33:07.321 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-mon[24602]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:07.321 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.037+0000 7f64f4362040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:33:07.321 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.098+0000 7f64f4362040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:33:07.322 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.253+0000 7f64f4362040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:33:07.322 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.322+0000 7f64f4362040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:33:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:07 smithi081 ceph-mon[23696]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:07 smithi081 ceph-mon[28146]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:07.621 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.383+0000 7f64f4362040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:33:07.621 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.556+0000 7f64f4362040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:33:07.621 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.622+0000 7f64f4362040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:33:07.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.689+0000 7f64f4362040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:33:07.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.781+0000 7f64f4362040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:33:07.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.841+0000 7f64f4362040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:33:08.200 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:07.903+0000 7f64f4362040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:33:08.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:08 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:08 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:08 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:08 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:08.838 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:08.732+0000 7f64f4362040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:33:08.838 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:08] ENGINE Bus STARTING 2024-09-17T01:33:08.838 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: CherryPy Checker: 2024-09-17T01:33:08.838 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: The Application mounted at '' has an empty config. 2024-09-17T01:33:08.838 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:09.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:08] ENGINE Serving on http://:::9283 2024-09-17T01:33:09.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:08] ENGINE Bus STARTED 2024-09-17T01:33:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: Standby manager daemon x restarted 2024-09-17T01:33:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:33:09.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:33:09.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:09.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:33:09.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:09 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: Standby manager daemon x restarted 2024-09-17T01:33:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: from='mgr.14412 172.21.15.81:0/341726112' entity='mgr.y' 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: Standby manager daemon x restarted 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:09.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:33:09.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:09 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:10.860 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:10 smithi081 ceph-mon[23696]: mgrmap e21: y(active, since 8m), standbys: x 2024-09-17T01:33:10.861 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:10 smithi081 ceph-mon[28146]: mgrmap e21: y(active, since 8m), standbys: x 2024-09-17T01:33:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:10 smithi105 ceph-mon[24602]: mgrmap e21: y(active, since 8m), standbys: x 2024-09-17T01:33:11.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:11 smithi105 ceph-mon[24602]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:11.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:11 smithi081 ceph-mon[23696]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:11.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:11 smithi081 ceph-mon[28146]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:12.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:12 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:12.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:33:12.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:12 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:12.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:12 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:13.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:13 smithi105 ceph-mon[24602]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:13 smithi081 ceph-mon[23696]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:13.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:13 smithi081 ceph-mon[28146]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:13.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:13] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-09-17T01:33:15.802 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:15 smithi105 ceph-mon[24602]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:15 smithi081 ceph-mon[23696]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:15.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:15 smithi081 ceph-mon[28146]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:17.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:17 smithi105 ceph-mon[24602]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:17.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:17 smithi081 ceph-mon[23696]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:17.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:17 smithi081 ceph-mon[28146]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:19.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:19 smithi105 ceph-mon[24602]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:19.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:19 smithi081 ceph-mon[23696]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:19.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:19 smithi081 ceph-mon[28146]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:21 smithi105 ceph-mon[24602]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:21.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:21 smithi081 ceph-mon[23696]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:21.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:21 smithi081 ceph-mon[28146]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:22.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:22 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:22.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:33:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:22 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:22 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:23 smithi105 ceph-mon[24602]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:23 smithi081 ceph-mon[23696]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:23] "GET /metrics HTTP/1.1" 200 207987 "" "Prometheus/2.33.4" 2024-09-17T01:33:23.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:23 smithi081 ceph-mon[28146]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:23.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:23.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:33:23.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:25.070 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:33:25.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:25 smithi081 ceph-mon[23696]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:25.677 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:25 smithi081 ceph-mon[28146]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:25.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:25 smithi105 ceph-mon[24602]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:26.992 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:33:26.992 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (6m) 19s ago 7m 25.8M - ba2b418f427c 1d46348ac778 2024-09-17T01:33:26.992 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (7m) 18s ago 7m 54.9M - 8.3.5 dad864ee21e9 d2a64aee5705 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (6m) 19s ago 6m 56.8M - 3.5 e1d6a67b021e d964d8155594 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (23s) 18s ago 11m 85.4M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:9283 running (13m) 19s ago 13m 462M - 17.2.0 e1d6a67b021e a454a71b7b66 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (13m) 19s ago 13m 52.0M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (12m) 18s ago 12m 41.5M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (12m) 19s ago 12m 38.4M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (8m) 19s ago 8m 16.9M - 1dbe0e931976 8b4df6dc354d 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (8m) 18s ago 8m 18.4M - 1dbe0e931976 8fa0015a15e5 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (11m) 19s ago 11m 50.9M 4016M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (11m) 19s ago 11m 51.6M 4016M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (10m) 19s ago 10m 47.7M 4016M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (10m) 19s ago 10m 50.7M 4016M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (9m) 18s ago 9m 50.0M 4272M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:33:26.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (9m) 18s ago 9m 49.3M 4272M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:33:26.994 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (9m) 18s ago 9m 48.7M 4272M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:33:26.994 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (8m) 18s ago 8m 49.1M 4272M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:33:26.994 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (6m) 18s ago 8m 67.2M - 514e6a882f6e 2b7eda538c73 2024-09-17T01:33:26.994 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (6m) 19s ago 6m 72.9M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:33:26.994 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (6m) 18s ago 6m 74.4M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:33:27.586 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:33:27.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:27 smithi105 ceph-mon[24602]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:27.883 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:27 smithi081 ceph-mon[23696]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:27.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:27 smithi081 ceph-mon[28146]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:28.862 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:28 smithi081 ceph-mon[23696]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:33:28.862 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:28 smithi081 ceph-mon[28146]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:33:28.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:28 smithi105 ceph-mon[24602]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:33:29.589 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:33:29.590 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:33:29.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:29 smithi105 ceph-mon[24602]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:29.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:29 smithi081 ceph-mon[23696]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:29.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:29 smithi081 ceph-mon[28146]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:30.216 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph -s' 2024-09-17T01:33:30.813 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:30 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2691367049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:30.814 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:30 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2691367049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:30.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:30 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2691367049' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:31.811 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:31 smithi081 ceph-mon[23696]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:31.811 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:31 smithi081 ceph-mon[28146]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:31.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:31 smithi105 ceph-mon[24602]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: cluster: 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: id: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: health: HEALTH_OK 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: services: 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: mgr: y(active, since 8m), standbys: x 2024-09-17T01:33:32.287 INFO:teuthology.orchestra.run.smithi081.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: data: 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: pools: 6 pools, 161 pgs 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: objects: 241 objects, 585 KiB 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: pgs: 161 active+clean 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: io: 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-17T01:33:32.288 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:33:32.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:32 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:32.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:32 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/799890830' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:33:32.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:33:32.908 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:32 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:32.908 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:32 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/799890830' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:33:32.908 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:32 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:32.908 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:32 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/799890830' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:33:32.931 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:33:33.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:33 smithi105 ceph-mon[24602]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:33.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-mon[28146]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:33.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-mon[23696]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:33.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:33] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-09-17T01:33:33.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=error ts=2024-09-17T01:33:33.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:33.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:33:33.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:34.902 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_OK 2024-09-17T01:33:35.536 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-09-17T01:33:35.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:35 smithi081 ceph-mon[23696]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:35.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:35 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1058621165' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:33:35.834 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:35 smithi081 ceph-mon[28146]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:35.834 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:35 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1058621165' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:33:35.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:35 smithi105 ceph-mon[24602]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:35.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:35 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1058621165' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:33:37.540 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:33:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:37 smithi105 ceph-mon[24602]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:37 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1303442822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:37.851 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:37 smithi081 ceph-mon[23696]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:37.851 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:37 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1303442822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:37.851 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:37 smithi081 ceph-mon[28146]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:33:37.851 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:37 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1303442822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:33:38.174 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph mgr fail' 2024-09-17T01:33:39.712 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:39 smithi081 ceph-mon[28146]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:39.712 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:39 smithi081 ceph-mon[23696]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:39 smithi105 ceph-mon[24602]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:41.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[23696]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:41.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:33:41.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[23696]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T01:33:41.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[28146]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:41.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:33:41.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mon[28146]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T01:33:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-mon[24602]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:33:41.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:33:41.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-mon[24602]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T01:33:41.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE Bus STOPPING 2024-09-17T01:33:41.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:33:41.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE Bus STOPPED 2024-09-17T01:33:41.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setuser ceph since I am not root 2024-09-17T01:33:41.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ignoring --setgroup ceph since I am not root 2024-09-17T01:33:41.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-mgr[23930]: -- 172.21.15.81:0/3362984843 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d6c9d82340 con 0x55d6c9d5b000 2024-09-17T01:33:41.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:41.300+0000 7f5bcfe3e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:33:41.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:41.379+0000 7f5bcfe3e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:33:41.773 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:41.555+0000 7f5bcfe3e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:33:41.773 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:41.649+0000 7f5bcfe3e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:33:41.796 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:33:42.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:41 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:41.783+0000 7f5bcfe3e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:33:42.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:42.011+0000 7f5bcfe3e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:33:42.132 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE Bus STARTING 2024-09-17T01:33:42.132 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE Serving on http://:::9283 2024-09-17T01:33:42.133 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:33:41] ENGINE Bus STARTED 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: mgrmap e22: x(active, starting, since 0.949396s) 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:33:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: Manager daemon x is now available 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: Queued rgw.foo for migration 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T01:33:42.360 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: Checking for cert/key for grafana.a 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: mgrmap e22: x(active, starting, since 0.949396s) 2024-09-17T01:33:42.361 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:33:42.362 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: Manager daemon x is now available 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: Queued rgw.foo for migration 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: Checking for cert/key for grafana.a 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.363 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:42.364 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.364 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.364 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.364 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3396441294' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: mgrmap e22: x(active, starting, since 0.949396s) 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:33:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: Manager daemon x is now available 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: Queued rgw.foo for migration 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: Migrating certs/keys for iscsi.foo spec to cert store 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: Migrating certs/keys for rgw.foo spec to cert store 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: Checking for cert/key for grafana.a 2024-09-17T01:33:42.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:42.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:42.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:33:42.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:33:42.626 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:42.360+0000 7f5bcfe3e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:33:42.626 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:42.459+0000 7f5bcfe3e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:33:42.627 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:42.628+0000 7f5bcfe3e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:33:43.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:42] "GET /metrics HTTP/1.1" 200 34899 "" "Prometheus/2.33.4" 2024-09-17T01:33:43.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[23696]: Deploying cephadm binary to smithi081 2024-09-17T01:33:43.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[23696]: Deploying cephadm binary to smithi105 2024-09-17T01:33:43.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[23696]: mgrmap e23: x(active, since 1.9511s) 2024-09-17T01:33:43.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[23696]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:43.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[28146]: Deploying cephadm binary to smithi081 2024-09-17T01:33:43.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[28146]: Deploying cephadm binary to smithi105 2024-09-17T01:33:43.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[28146]: mgrmap e23: x(active, since 1.9511s) 2024-09-17T01:33:43.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-mon[28146]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:43 smithi105 ceph-mon[24602]: Deploying cephadm binary to smithi081 2024-09-17T01:33:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:43 smithi105 ceph-mon[24602]: Deploying cephadm binary to smithi105 2024-09-17T01:33:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:43 smithi105 ceph-mon[24602]: mgrmap e23: x(active, since 1.9511s) 2024-09-17T01:33:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:43 smithi105 ceph-mon[24602]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:43.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=error ts=2024-09-17T01:33:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:43.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:33:43.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:44.138 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:43.730+0000 7f5bcfe3e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:33:44.138 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:43.826+0000 7f5bcfe3e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[23696]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[23696]: mgrmap e24: x(active, since 2s) 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[28146]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:44.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[28146]: mgrmap e24: x(active, since 2s) 2024-09-17T01:33:44.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.472 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-mon[24602]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:44.472 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-mon[24602]: mgrmap e24: x(active, since 2s) 2024-09-17T01:33:44.472 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.472 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:44.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:44.649+0000 7f644cb14640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:01:33:44] ENGINE Error in HTTPServer.serve 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._connections.run(self.expiration_interval) 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._run(expiration_interval) 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: s = self.context.wrap_socket( 2024-09-17T01:33:44.886 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.sslsocket_class._create( 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.do_handshake() 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._sslobj.do_handshake() 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:33:44.887 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:33:45.128 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:44.783+0000 7f5bcfe3e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:33:45.128 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.130+0000 7f5bcfe3e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:33:45.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:45 smithi105 ceph-mon[24602]: [17/Sep/2024:01:33:44] ENGINE Bus STARTING 2024-09-17T01:33:45.421 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-mon[23696]: [17/Sep/2024:01:33:44] ENGINE Bus STARTING 2024-09-17T01:33:45.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-mon[28146]: [17/Sep/2024:01:33:44] ENGINE Bus STARTING 2024-09-17T01:33:45.421 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.269+0000 7f5bcfe3e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:33:45.421 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.365+0000 7f5bcfe3e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:33:45.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.466+0000 7f5bcfe3e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:33:45.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.557+0000 7f5bcfe3e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:33:46.097 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.714+0000 7f5bcfe3e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:33:46.097 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.816+0000 7f5bcfe3e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:33:46.097 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:45.916+0000 7f5bcfe3e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:33:46.389 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:46.210+0000 7f5bcfe3e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:33:46.389 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:46.299+0000 7f5bcfe3e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:33:46.389 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:46.391+0000 7f5bcfe3e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: [17/Sep/2024:01:33:44] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: [17/Sep/2024:01:33:44] ENGINE Error in HTTPServer.serve 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:33:46.390 INFO:journalctl@ceph.mon.a.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.do_handshake() 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: [17/Sep/2024:01:33:44] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: [17/Sep/2024:01:33:44] ENGINE Bus STARTED 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[23696]: mgrmap e25: x(active, since 4s) 2024-09-17T01:33:46.391 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: [17/Sep/2024:01:33:44] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: [17/Sep/2024:01:33:44] ENGINE Error in HTTPServer.serve 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:33:46.392 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.do_handshake() 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: [17/Sep/2024:01:33:44] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: [17/Sep/2024:01:33:44] ENGINE Bus STARTED 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:46.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:46 smithi081 ceph-mon[28146]: mgrmap e25: x(active, since 4s) 2024-09-17T01:33:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: [17/Sep/2024:01:33:44] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:33:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: [17/Sep/2024:01:33:44] ENGINE Error in HTTPServer.serve 2024-09-17T01:33:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:33:46.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: [17/Sep/2024:01:33:44] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: [17/Sep/2024:01:33:44] ENGINE Bus STARTED 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:46.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:46 smithi105 ceph-mon[24602]: mgrmap e25: x(active, since 4s) 2024-09-17T01:33:47.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:47.527+0000 7f5bcfe3e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:33:47.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:33:47] ENGINE Bus STARTING 2024-09-17T01:33:47.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: CherryPy Checker: 2024-09-17T01:33:47.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: The Application mounted at '' has an empty config. 2024-09-17T01:33:47.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: 2024-09-17T01:33:47.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:33:47] ENGINE Serving on http://:::9283 2024-09-17T01:33:47.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: [17/Sep/2024:01:33:47] ENGINE Bus STARTED 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: Standby manager daemon y started 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:48.411 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: Standby manager daemon y started 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.412 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.413 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.413 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: Standby manager daemon y started 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:33:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1690468325' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:33:48.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:48.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:49.324Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:33:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-17T01:33:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:33:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi105 to 3248M 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: Adjusting osd_memory_target on smithi081 to 2992M 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:33:49.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi105 to 3248M 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: Adjusting osd_memory_target on smithi081 to 2992M 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:33:49.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:49 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:33:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: mgrmap e26: x(active, since 7s), standbys: y 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi105 to 3248M 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: Adjusting osd_memory_target on smithi081 to 2992M 2024-09-17T01:33:49.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:49.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:33:49.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:33:49.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:33:49.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:49 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:33:50.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 systemd[1]: Stopping Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:33:50.667 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 podman[56888]: 2024-09-17 01:33:50.372208627 +0000 UTC m=+0.135298772 container died 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:33:50.667 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T01:33:50.668 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[23696]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T01:33:50.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:50 smithi081 ceph-mon[28146]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:33:50.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:33:50.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T01:33:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:50 smithi105 ceph-mon[24602]: Deploying daemon node-exporter.a on smithi081 2024-09-17T01:33:50.930 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 podman[56888]: 2024-09-17 01:33:50.669964954 +0000 UTC m=+0.433055095 container cleanup 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:50.930 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 bash[56888]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a 2024-09-17T01:33:50.930 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 podman[56900]: 2024-09-17 01:33:50.888045188 +0000 UTC m=+0.514196984 container remove 8b4df6dc354d96ce7b517695f0e1eef16c0bce992fcc9207750806b44a21ab20 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:50.930 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:50 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:33:51.425 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-17T01:33:51.425 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 systemd[1]: Stopped Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:33:51.426 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 4.155s CPU time. 2024-09-17T01:33:51.426 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 systemd[1]: Starting Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:33:51.680 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 bash[57028]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-17T01:33:52.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 bash[57028]: Getting image source signatures 2024-09-17T01:33:52.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 bash[57028]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-17T01:33:52.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 bash[57028]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-17T01:33:52.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:51 smithi081 bash[57028]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-17T01:33:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:52 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:52 smithi105 ceph-mon[24602]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:33:52.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:33:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:52] "GET /metrics HTTP/1.1" 200 34899 "" "Prometheus/2.33.4" 2024-09-17T01:33:52.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:52 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:52.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:52 smithi081 ceph-mon[23696]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:33:52.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:52 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:33:52.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:52 smithi081 ceph-mon[28146]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:33:53.507 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:53 smithi081 bash[57028]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-17T01:33:53.507 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:53 smithi081 bash[57028]: Writing manifest to image destination 2024-09-17T01:33:53.877 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:53 smithi081 podman[57028]: 2024-09-17 01:33:53.632755816 +0000 UTC m=+2.296705835 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-17T01:33:53.877 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:53 smithi081 podman[57028]: 2024-09-17 01:33:53.756358752 +0000 UTC m=+2.420308758 container create 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:53.877 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:33:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:33:53.878 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=error ts=2024-09-17T01:33:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:53.878 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.81:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.81 because it doesn't contain any IP SANs" 2024-09-17T01:33:53.878 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:33:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=warn ts=2024-09-17T01:33:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.105:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.105 because it doesn't contain any IP SANs" 2024-09-17T01:33:54.430 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 podman[57028]: 2024-09-17 01:33:54.008404583 +0000 UTC m=+2.672354598 container init 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:54.430 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 podman[57028]: 2024-09-17 01:33:54.011367089 +0000 UTC m=+2.675317097 container start 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T01:33:54.430 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.015Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.015Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=arp 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-17T01:33:54.431 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=edac 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-17T01:33:54.432 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-17T01:33:54.433 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=os 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=stat 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-17T01:33:54.434 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=time 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=uname 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a[57103]: ts=2024-09-17T01:33:54.016Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-17T01:33:54.435 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 bash[57028]: 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb 2024-09-17T01:33:54.436 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 01:33:54 smithi081 systemd[1]: Started Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:33:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:54 smithi105 ceph-mon[24602]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:33:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:54 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:54 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[28146]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[23696]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:54 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:33:55.786 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:55 smithi105 ceph-mon[24602]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:33:55.787 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:55 smithi105 ceph-mon[24602]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:33:55.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:55 smithi081 ceph-mon[23696]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:33:55.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:55 smithi081 ceph-mon[23696]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:33:55.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:55 smithi081 ceph-mon[28146]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T01:33:55.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:55 smithi081 ceph-mon[28146]: Deploying daemon alertmanager.a on smithi081 2024-09-17T01:33:56.814 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:56 smithi105 ceph-mon[24602]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:33:56.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:56 smithi081 ceph-mon[23696]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:33:56.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:56 smithi081 ceph-mon[28146]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:33:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:57.892 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:57.893 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:33:58.652 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:33:58 smithi081 ceph-mon[23696]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:33:58.652 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:33:58 smithi081 ceph-mon[28146]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:33:58.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:33:58 smithi105 ceph-mon[24602]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:34:00.708 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:00 smithi081 ceph-mon[23696]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:34:00.708 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:00 smithi081 ceph-mon[28146]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:34:00.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:00 smithi105 ceph-mon[24602]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:34:01.339 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 systemd[1]: Stopping Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:01.600 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[47866]: level=info ts=2024-09-17T01:34:01.341Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:34:01.603 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 podman[57453]: 2024-09-17 01:34:01.362132111 +0000 UTC m=+0.132841749 container died 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:01.909 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 podman[57453]: 2024-09-17 01:34:01.692567889 +0000 UTC m=+0.463277530 container cleanup 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:01.909 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 bash[57453]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a 2024-09-17T01:34:02.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 podman[57465]: 2024-09-17 01:34:01.911474306 +0000 UTC m=+0.543811413 container remove 1d46348ac7781194cd3abadd6db3feafc32010e9b0cea1aedad2f8782b15f0ba (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:02.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:01 smithi081 podman[57465]: 2024-09-17 01:34:01.987010491 +0000 UTC m=+0.619347553 volume remove 20151da017a9e094ad6353c970a4ff36ba592a1ffac1ebf5690475fe8b055bad 2024-09-17T01:34:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:02 smithi105 ceph-mon[24602]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:02.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:02 smithi105 ceph-mon[24602]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:02.637 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:02.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.81:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.105:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:02.653 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:02 smithi081 ceph-mon[23696]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:02.653 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:02 smithi081 ceph-mon[23696]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:02.654 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:02 smithi081 ceph-mon[28146]: from='client.24700 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:02.654 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:02 smithi081 ceph-mon[28146]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:02.654 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T01:34:02.654 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 systemd[1]: Stopped Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:02.654 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.245s CPU time. 2024-09-17T01:34:02.655 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 systemd[1]: Starting Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:02.915 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 podman[57596]: 2024-09-17 01:34:02.656052652 +0000 UTC m=+0.095245825 volume create dfd5d2ed5e046d7af1faa50307c1d23768169b84640a61be422da6fb8ecb4d97 2024-09-17T01:34:02.915 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 podman[57596]: 2024-09-17 01:34:02.577302054 +0000 UTC m=+0.016495229 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-17T01:34:02.915 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:02 smithi081 podman[57596]: 2024-09-17 01:34:02.756624633 +0000 UTC m=+0.195817801 container create dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:03.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:02] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 podman[57596]: 2024-09-17 01:34:03.045409087 +0000 UTC m=+0.484602265 container init dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 podman[57596]: 2024-09-17 01:34:03.048197003 +0000 UTC m=+0.487390177 container start dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.060Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.060Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.061Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.81 port=9094 2024-09-17T01:34:03.180 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.062Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T01:34:03.181 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.087Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:34:03.181 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.087Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T01:34:03.181 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.089Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-17T01:34:03.181 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:03.089Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-17T01:34:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:34:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:34:03.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 bash[57596]: dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d 2024-09-17T01:34:03.931 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:03 smithi081 systemd[1]: Started Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:04.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:04.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:04 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:04.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:04.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:04.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:04.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:34:04.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:04 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:05.425 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:05.062Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000174217s 2024-09-17T01:34:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:05 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:34:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:05 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:34:05.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:05 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:34:05.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:05 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:34:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:05 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:34:05.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:05 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:34:06.707 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:06 smithi081 ceph-mon[23696]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:06.708 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:06 smithi081 ceph-mon[28146]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:06.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:06 smithi105 ceph-mon[24602]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:08.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:08 smithi105 ceph-mon[24602]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:08.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:08 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:08 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:08 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/372228871' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:34:08.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[23696]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:08.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/372228871' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:34:08.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[28146]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:08.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:08.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:08 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/372228871' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:34:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:09 smithi105 ceph-mon[24602]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:34:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:09 smithi105 ceph-mon[24602]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:34:09.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:09 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]: dispatch 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[23696]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[23696]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]: dispatch 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[28146]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[28146]: Deploying daemon prometheus.a on smithi105 2024-09-17T01:34:09.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:09 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]: dispatch 2024-09-17T01:34:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:10 smithi105 ceph-mon[24602]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:10 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]': finished 2024-09-17T01:34:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:10 smithi105 ceph-mon[24602]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T01:34:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:10 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3606070517' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:10.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:10 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:10.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[23696]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:10.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]': finished 2024-09-17T01:34:10.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[23696]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T01:34:10.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3606070517' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:10.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:10.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[28146]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:10.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2073268307' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/52805828"}]': finished 2024-09-17T01:34:10.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[28146]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T01:34:10.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3606070517' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:10.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:10 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]: dispatch 2024-09-17T01:34:11.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:11 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]': finished 2024-09-17T01:34:11.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:11 smithi105 ceph-mon[24602]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T01:34:11.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:11 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]: dispatch 2024-09-17T01:34:11.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]': finished 2024-09-17T01:34:11.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[23696]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T01:34:11.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]: dispatch 2024-09-17T01:34:11.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/52805828"}]': finished 2024-09-17T01:34:11.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[28146]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T01:34:11.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:11 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]: dispatch 2024-09-17T01:34:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:34:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]': finished 2024-09-17T01:34:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T01:34:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/741317590' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:12.636 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.81:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.105:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:12.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:34:12.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]': finished 2024-09-17T01:34:12.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/741317590' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2674901354' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/4171850412"}]': finished 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/741317590' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:12.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:12 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]: dispatch 2024-09-17T01:34:13.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:12] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-09-17T01:34:13.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:34:13.066Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003349646s 2024-09-17T01:34:13.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:13 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]': finished 2024-09-17T01:34:13.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:13 smithi105 ceph-mon[24602]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T01:34:13.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:13 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]: dispatch 2024-09-17T01:34:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[23926]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-09-17T01:34:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]': finished 2024-09-17T01:34:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[23696]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T01:34:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]: dispatch 2024-09-17T01:34:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3230294174"}]': finished 2024-09-17T01:34:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[28146]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T01:34:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:13 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]: dispatch 2024-09-17T01:34:14.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:14 smithi105 ceph-mon[24602]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:14.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]': finished 2024-09-17T01:34:14.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:14 smithi105 ceph-mon[24602]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T01:34:14.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/99962972' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:14.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:14 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:14.903 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[23696]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]': finished 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[23696]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/99962972' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[28146]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3915076907' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1237785981"}]': finished 2024-09-17T01:34:14.904 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[28146]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T01:34:14.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/99962972' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:14.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:14 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]: dispatch 2024-09-17T01:34:15.854 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:15 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]': finished 2024-09-17T01:34:15.854 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:15 smithi105 ceph-mon[24602]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T01:34:15.854 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:15 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1931583165' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:15.854 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:15 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]': finished 2024-09-17T01:34:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[23696]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T01:34:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1931583165' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:15.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:15.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/62530244"}]': finished 2024-09-17T01:34:15.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[28146]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T01:34:15.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1931583165' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:15.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:15 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]: dispatch 2024-09-17T01:34:16.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:16 smithi105 ceph-mon[24602]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:16.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:16 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]': finished 2024-09-17T01:34:16.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:16 smithi105 ceph-mon[24602]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T01:34:16.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[28146]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:16.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]': finished 2024-09-17T01:34:16.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[28146]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T01:34:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[23696]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1770244867"}]': finished 2024-09-17T01:34:16.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:16 smithi081 ceph-mon[23696]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T01:34:18.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:18 smithi105 ceph-mon[24602]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:34:18.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:18 smithi081 ceph-mon[23696]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:34:18.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:18 smithi081 ceph-mon[28146]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:34:19.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:18 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T01:34:19.469 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T01:34:19.470 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-09-17T01:34:19.470 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[41487]: ts=2024-09-17T01:34:19.188Z caller=main.go:1066 level=info msg="See you next time!" 2024-09-17T01:34:19.470 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 podman[47392]: 2024-09-17 01:34:19.198327112 +0000 UTC m=+0.101597343 container died 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:19.835 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:19.836 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 podman[47392]: 2024-09-17 01:34:19.570751287 +0000 UTC m=+0.474021510 container cleanup 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:19.836 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 bash[47392]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T01:34:20.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:19 smithi105 podman[47407]: 2024-09-17 01:34:19.838615212 +0000 UTC m=+0.642119340 container remove 2b7eda538c7342b583bd1f392090c6bcee3884b7a3c41e4409775889a0e3222b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:20.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T01:34:20.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 systemd[1]: Stopped Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:20.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.372s CPU time. 2024-09-17T01:34:20.545 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 systemd[1]: Starting Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:20.545 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 podman[47534]: 2024-09-17 01:34:20.315167506 +0000 UTC m=+0.016730807 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-17T01:34:20.545 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 podman[47534]: 2024-09-17 01:34:20.423297125 +0000 UTC m=+0.124860415 container create 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:20.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-mon[24602]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 podman[47534]: 2024-09-17 01:34:20.667777963 +0000 UTC m=+0.369341257 container init 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 podman[47534]: 2024-09-17 01:34:20.670853574 +0000 UTC m=+0.372416866 container start 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.693Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.693Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.693Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.693Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.693Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T01:34:20.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.695Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.696Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.697Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.697Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.699Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.699Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.623µs 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.699Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.706Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.722Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.722Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.722Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.049µs wal_replay_duration=23.024014ms wbl_replay_duration=85ns total_replay_duration=23.062018ms 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.724Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T01:34:20.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.724Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.724Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.744Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.466729ms db_storage=16.816µs remote_storage=1.437µs web_handler=380ns query_engine=626ns scrape=449.238µs scrape_sd=97.629µs notify=9.441µs notify_sd=9.409µs rules=18.499828ms tracing=5.49µs 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.744Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:20.744Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 bash[47534]: 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 2024-09-17T01:34:20.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:20 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:20.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:20 smithi081 ceph-mon[23696]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:20.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:20 smithi081 ceph-mon[28146]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:21.629 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:21 smithi105 systemd[1]: Stopping Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:21.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:21 smithi105 podman[47712]: 2024-09-17 01:34:21.632438536 +0000 UTC m=+0.134993165 container died 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:21.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:21 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:21.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:21 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:21.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:21 smithi105 ceph-mon[24602]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T01:34:21.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:21 smithi105 ceph-mon[24602]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:34:21.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:21 smithi105 ceph-mon[24602]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 673 B/s rd, 0 op/s 2024-09-17T01:34:22.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[23696]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[23696]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[23696]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 673 B/s rd, 0 op/s 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[28146]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[28146]: Deploying daemon node-exporter.b on smithi105 2024-09-17T01:34:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:21 smithi081 ceph-mon[28146]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 673 B/s rd, 0 op/s 2024-09-17T01:34:22.237 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 podman[47712]: 2024-09-17 01:34:22.030580522 +0000 UTC m=+0.533135146 container cleanup 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:22.237 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 bash[47712]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b 2024-09-17T01:34:22.599 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 podman[47725]: 2024-09-17 01:34:22.240091927 +0000 UTC m=+0.611464122 container remove 8fa0015a15e534fdaab89e18be024735bf8982c19089a821d640199ab4da334e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:22.599 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:34:22.599 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-17T01:34:22.599 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 systemd[1]: Stopped Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:22.599 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 4.377s CPU time. 2024-09-17T01:34:22.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 systemd[1]: Starting Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:22.885 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:22 smithi105 bash[47855]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-17T01:34:23.571 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:23 smithi105 bash[47855]: Getting image source signatures 2024-09-17T01:34:23.571 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:23 smithi105 bash[47855]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-17T01:34:23.571 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:23 smithi105 bash[47855]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-17T01:34:23.571 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:23 smithi105 bash[47855]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-17T01:34:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:24 smithi105 ceph-mon[24602]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:24 smithi081 ceph-mon[23696]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:24 smithi081 ceph-mon[28146]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:26 smithi105 ceph-mon[24602]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:34:26.385 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:26 smithi105 bash[47855]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-17T01:34:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:26 smithi081 ceph-mon[23696]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:34:26.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:26 smithi081 ceph-mon[28146]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:34:26.822 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:26 smithi105 bash[47855]: Writing manifest to image destination 2024-09-17T01:34:27.131 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:26 smithi105 podman[47855]: 2024-09-17 01:34:26.753223384 +0000 UTC m=+4.054357281 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-17T01:34:27.131 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:26 smithi105 podman[47855]: 2024-09-17 01:34:26.910476328 +0000 UTC m=+4.211610224 container create 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 podman[47855]: 2024-09-17 01:34:27.145942549 +0000 UTC m=+4.447076444 container init 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 podman[47855]: 2024-09-17 01:34:27.14898236 +0000 UTC m=+4.450116255 container start 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.152Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.152Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.153Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.153Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.153Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.153Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T01:34:27.386 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=arp 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-17T01:34:27.387 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=edac 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-17T01:34:27.388 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-17T01:34:27.389 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-17T01:34:27.389 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-17T01:34:27.389 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-17T01:34:27.389 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-17T01:34:27.389 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=os 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-17T01:34:27.390 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=stat 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=time 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=uname 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-17T01:34:27.391 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-17T01:34:27.392 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-17T01:34:27.392 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-17T01:34:27.392 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b[47940]: ts=2024-09-17T01:34:27.154Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-17T01:34:27.392 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 bash[47855]: 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 2024-09-17T01:34:27.392 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 01:34:27 smithi105 systemd[1]: Started Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:34:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: Reconfiguring grafana.a (dependencies changed)... 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:28 smithi105 ceph-mon[24602]: Reconfiguring daemon grafana.a on smithi105 2024-09-17T01:34:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:34:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: Reconfiguring grafana.a (dependencies changed)... 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[23696]: Reconfiguring daemon grafana.a on smithi105 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:34:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: Reconfiguring grafana.a (dependencies changed)... 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:34:28.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:28 smithi081 ceph-mon[28146]: Reconfiguring daemon grafana.a on smithi105 2024-09-17T01:34:29.368 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:30.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:30 smithi105 ceph-mon[24602]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:30.636 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 systemd[1]: Stopping Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:30.636 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[40709]: t=2024-09-17T01:34:30+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-17T01:34:30.636 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 podman[48114]: 2024-09-17 01:34:30.409831143 +0000 UTC m=+0.141909973 container died d2a64aee5705c20cc4afe53bc35d51383502ab3f10874d6ae0580ba0fc0417a2 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.buildah.version=1.24.2, release=236.1648460182, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-09-17T01:34:30.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:30 smithi081 ceph-mon[23696]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:30.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:30 smithi081 ceph-mon[28146]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:31.022 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 podman[48114]: 2024-09-17 01:34:30.781999399 +0000 UTC m=+0.514078232 container cleanup d2a64aee5705c20cc4afe53bc35d51383502ab3f10874d6ae0580ba0fc0417a2 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, io.openshift.tags=base rhel8, release=236.1648460182, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.buildah.version=1.24.2, maintainer=Paul Cuzner ) 2024-09-17T01:34:31.022 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 bash[48114]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a 2024-09-17T01:34:31.022 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:30 smithi105 bash[48141]: Error: no container with name or ID "ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana.a" found: no such container 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 podman[48126]: 2024-09-17 01:34:31.024886315 +0000 UTC m=+0.613818474 container remove d2a64aee5705c20cc4afe53bc35d51383502ab3f10874d6ae0580ba0fc0417a2 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.component=ubi8-container, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 systemd[1]: Stopped Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Consumed 3.363s CPU time. 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 systemd[1]: Starting Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:34:31.273 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 podman[48205]: 2024-09-17 01:34:31.155185387 +0000 UTC m=+0.015605250 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-09-17T01:34:31.274 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 podman[48205]: 2024-09-17 01:34:31.275978026 +0000 UTC m=+0.136397938 container create df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, io.openshift.expose-services=, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, maintainer=Paul Cuzner , architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.buildah.version=1.24.2, distribution-scope=public, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Bus STOPPING 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Bus STOPPED 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Bus STARTING 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Serving on http://:::9283 2024-09-17T01:34:31.813 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Bus STARTED 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 podman[48205]: 2024-09-17 01:34:31.536676873 +0000 UTC m=+0.397096725 container init df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, release=236.1648460182, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., name=ubi8, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , description=Ceph Grafana Container, io.buildah.version=1.24.2) 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 podman[48205]: 2024-09-17 01:34:31.539476722 +0000 UTC m=+0.399896572 container start df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2) 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-17T01:34:31.814 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="App mode production" logger=settings 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-09-17T01:34:31.815 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=591.342µs 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 bash[48205]: df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 systemd[1]: Started Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-17T01:34:31.816 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-09-17T01:34:31.817 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-09-17T01:34:31.817 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-09-17T01:34:31.817 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:34:31+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T01:34:32.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:31] ENGINE Bus STOPPING 2024-09-17T01:34:32.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:34:32.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STOPPED 2024-09-17T01:34:32.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STARTING 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Serving on http://:::9283 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STARTED 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STOPPING 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:32.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:32.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:32 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:33.284 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:34:33.284 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STOPPED 2024-09-17T01:34:33.284 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:32] ENGINE Bus STARTING 2024-09-17T01:34:33.284 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:33] ENGINE Serving on http://:::9283 2024-09-17T01:34:33.284 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:34:33] ENGINE Bus STARTED 2024-09-17T01:34:33.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:33] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:33.637 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi081.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: Adding iSCSI gateway http://:@172.21.15.81:5000 to Dashboard 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi081"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:34:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:34:33.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:34:33.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:34:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:34 smithi105 ceph-mon[24602]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:34 smithi081 ceph-mon[28146]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:34 smithi081 ceph-mon[23696]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:36.429 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:34:36.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:36 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:34:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:34:36.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:36 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:34:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:38 smithi105 ceph-mon[24602]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:38 smithi081 ceph-mon[23696]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:38.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:38 smithi081 ceph-mon[28146]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:39.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:40 smithi105 ceph-mon[24602]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:40.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:40 smithi081 ceph-mon[23696]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:40 smithi081 ceph-mon[28146]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:42 smithi105 ceph-mon[24602]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:42 smithi081 ceph-mon[23696]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:42 smithi081 ceph-mon[28146]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:43] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-17T01:34:43.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:44 smithi105 ceph-mon[24602]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:44 smithi081 ceph-mon[23696]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:44 smithi081 ceph-mon[28146]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:46 smithi105 ceph-mon[24602]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:46 smithi081 ceph-mon[23696]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:46.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:46 smithi081 ceph-mon[28146]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:48 smithi105 ceph-mon[24602]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:48.650 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:48 smithi081 ceph-mon[23696]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:48.650 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:48 smithi081 ceph-mon[28146]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:49.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:50.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:50 smithi105 ceph-mon[24602]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:50.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:50 smithi081 ceph-mon[23696]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:50.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:50 smithi081 ceph-mon[28146]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:52 smithi105 ceph-mon[24602]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:52 smithi081 ceph-mon[23696]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:52.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:52 smithi081 ceph-mon[28146]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:34:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:34:53] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.43.0" 2024-09-17T01:34:53.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:54 smithi105 ceph-mon[24602]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:54 smithi081 ceph-mon[23696]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:54 smithi081 ceph-mon[28146]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:56 smithi081 ceph-mon[23696]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:56 smithi081 ceph-mon[28146]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:56.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:56 smithi105 ceph-mon[24602]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:34:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:57.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:34:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:58 smithi105 ceph-mon[24602]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:58.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:58 smithi081 ceph-mon[23696]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:58.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:58 smithi081 ceph-mon[28146]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:34:58.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:34:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:34:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:34:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:34:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:34:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:34:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:34:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:00 smithi081 ceph-mon[23696]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:00.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:00 smithi081 ceph-mon[28146]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:00 smithi105 ceph-mon[24602]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:02 smithi081 ceph-mon[23696]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:02.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:02 smithi081 ceph-mon[28146]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:02 smithi105 ceph-mon[24602]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:03] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.43.0" 2024-09-17T01:35:03.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:04.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:04 smithi081 ceph-mon[23696]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:04.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:04 smithi081 ceph-mon[28146]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:04 smithi105 ceph-mon[24602]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:06 smithi081 ceph-mon[23696]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:06 smithi081 ceph-mon[28146]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:06 smithi105 ceph-mon[24602]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:08 smithi105 ceph-mon[24602]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:08 smithi081 ceph-mon[23696]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:08 smithi081 ceph-mon[28146]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:09.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:10 smithi105 ceph-mon[24602]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:10 smithi081 ceph-mon[23696]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:10.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:10 smithi081 ceph-mon[28146]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:12 smithi105 ceph-mon[24602]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:12 smithi081 ceph-mon[23696]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:12 smithi081 ceph-mon[28146]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:13.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:13] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-17T01:35:13.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:14 smithi105 ceph-mon[24602]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:14 smithi081 ceph-mon[23696]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:14 smithi081 ceph-mon[28146]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:16 smithi081 ceph-mon[23696]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:16 smithi081 ceph-mon[28146]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:16 smithi105 ceph-mon[24602]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:18 smithi105 ceph-mon[24602]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:18 smithi081 ceph-mon[23696]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:18.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:18 smithi081 ceph-mon[28146]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:19.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:20 smithi105 ceph-mon[24602]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:20.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:20 smithi081 ceph-mon[23696]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:20 smithi081 ceph-mon[28146]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:22 smithi105 ceph-mon[24602]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:22 smithi081 ceph-mon[23696]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:22 smithi081 ceph-mon[28146]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:23.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:23] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.43.0" 2024-09-17T01:35:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:24 smithi105 ceph-mon[24602]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:24 smithi081 ceph-mon[23696]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:24 smithi081 ceph-mon[28146]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:26 smithi081 ceph-mon[23696]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:26.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:26 smithi081 ceph-mon[28146]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:26.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:26 smithi105 ceph-mon[24602]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:28.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:28 smithi081 ceph-mon[23696]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:28 smithi081 ceph-mon[28146]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:28 smithi105 ceph-mon[24602]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:29.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:30 smithi081 ceph-mon[23696]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:30.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:30 smithi081 ceph-mon[28146]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:30 smithi105 ceph-mon[24602]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:32 smithi105 ceph-mon[24602]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:32 smithi081 ceph-mon[23696]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:32 smithi081 ceph-mon[28146]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:33.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:33] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.43.0" 2024-09-17T01:35:33.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:34 smithi105 ceph-mon[24602]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:34 smithi081 ceph-mon[23696]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:34 smithi081 ceph-mon[28146]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:36 smithi081 ceph-mon[23696]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:36 smithi081 ceph-mon[28146]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:36 smithi105 ceph-mon[24602]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:38.338 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:38 smithi081 ceph-mon[23696]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:38.339 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:38 smithi081 ceph-mon[28146]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:38 smithi105 ceph-mon[24602]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:38.990 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:40 smithi105 ceph-mon[24602]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:40 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:40 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:40 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:40 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[28146]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[23696]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:40.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:40.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:40 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:41 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:35:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:41 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:35:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:41 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:35:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:35:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:35:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:35:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:41 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:35:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:42 smithi105 ceph-mon[24602]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:42.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:42 smithi081 ceph-mon[23696]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:42.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:42 smithi081 ceph-mon[28146]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:42.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:43] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-17T01:35:43.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:44.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:44 smithi105 ceph-mon[24602]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:44.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:44 smithi081 ceph-mon[23696]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:44.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:44 smithi081 ceph-mon[28146]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:46 smithi081 ceph-mon[23696]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:46.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:46 smithi081 ceph-mon[28146]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:46 smithi105 ceph-mon[24602]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:48.248 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:48 smithi081 ceph-mon[23696]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:48.248 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:48 smithi081 ceph-mon[28146]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:48 smithi105 ceph-mon[24602]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:49.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:50.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:50 smithi105 ceph-mon[24602]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:50.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:50 smithi081 ceph-mon[23696]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:50.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:50 smithi081 ceph-mon[28146]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:52 smithi105 ceph-mon[24602]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:52 smithi081 ceph-mon[23696]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:52.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:52 smithi081 ceph-mon[28146]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:35:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:35:53] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.43.0" 2024-09-17T01:35:53.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:54 smithi105 ceph-mon[24602]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:54 smithi081 ceph-mon[23696]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:54 smithi081 ceph-mon[28146]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:56 smithi081 ceph-mon[23696]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:56 smithi081 ceph-mon[28146]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:56.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:56 smithi105 ceph-mon[24602]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:35:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:57.625 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:35:58.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:58 smithi081 ceph-mon[23696]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:58.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:58 smithi081 ceph-mon[28146]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:58 smithi105 ceph-mon[24602]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:35:59.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:35:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:35:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:35:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:35:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:35:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:35:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:35:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:00 smithi081 ceph-mon[23696]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:00.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:00 smithi081 ceph-mon[28146]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:00 smithi105 ceph-mon[24602]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:02 smithi105 ceph-mon[24602]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:02 smithi081 ceph-mon[23696]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:02 smithi081 ceph-mon[28146]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:03] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:36:03.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:04 smithi105 ceph-mon[24602]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:04 smithi081 ceph-mon[23696]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:04 smithi081 ceph-mon[28146]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:06 smithi081 ceph-mon[23696]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:06 smithi081 ceph-mon[28146]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:06 smithi105 ceph-mon[24602]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:08 smithi081 ceph-mon[23696]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:08.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:08 smithi081 ceph-mon[28146]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:08 smithi105 ceph-mon[24602]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:09.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:10 smithi105 ceph-mon[24602]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:10 smithi081 ceph-mon[23696]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:10.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:10 smithi081 ceph-mon[28146]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:12 smithi105 ceph-mon[24602]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:12 smithi081 ceph-mon[23696]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:12 smithi081 ceph-mon[28146]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:13.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:13] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:36:13.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:14 smithi105 ceph-mon[24602]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:14 smithi081 ceph-mon[23696]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:14 smithi081 ceph-mon[28146]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:16 smithi081 ceph-mon[23696]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:16 smithi081 ceph-mon[28146]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:16 smithi105 ceph-mon[24602]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:18 smithi081 ceph-mon[23696]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:18 smithi081 ceph-mon[28146]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:18 smithi105 ceph-mon[24602]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:19.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:20 smithi105 ceph-mon[24602]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:20.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:20 smithi081 ceph-mon[23696]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:20 smithi081 ceph-mon[28146]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:22 smithi105 ceph-mon[24602]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:22 smithi081 ceph-mon[23696]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:22 smithi081 ceph-mon[28146]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:23.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:23] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:36:23.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:24 smithi105 ceph-mon[24602]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:24 smithi081 ceph-mon[23696]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:24 smithi081 ceph-mon[28146]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:26 smithi081 ceph-mon[23696]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:26.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:26 smithi081 ceph-mon[28146]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:26.436 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:26 smithi105 ceph-mon[24602]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:28.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:28 smithi081 ceph-mon[23696]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:28 smithi081 ceph-mon[28146]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:28 smithi105 ceph-mon[24602]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:29.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:30 smithi081 ceph-mon[23696]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:30.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:30 smithi081 ceph-mon[28146]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:30 smithi105 ceph-mon[24602]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:32 smithi105 ceph-mon[24602]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:32 smithi081 ceph-mon[23696]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:32 smithi081 ceph-mon[28146]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:33.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:33] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-17T01:36:33.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:34 smithi105 ceph-mon[24602]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:34 smithi081 ceph-mon[23696]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:34 smithi081 ceph-mon[28146]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:36 smithi081 ceph-mon[23696]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:36 smithi081 ceph-mon[28146]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:36 smithi105 ceph-mon[24602]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:38 smithi081 ceph-mon[23696]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:38 smithi081 ceph-mon[28146]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:38 smithi105 ceph-mon[24602]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:39.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:40.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:40 smithi105 ceph-mon[24602]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:40.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:40 smithi081 ceph-mon[23696]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:40 smithi081 ceph-mon[28146]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:42.358 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:42 smithi081 ceph-mon[23696]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:42.359 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:42 smithi081 ceph-mon[28146]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:42.359 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:42 smithi105 ceph-mon[24602]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:43] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-17T01:36:43.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:44.548 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:44 smithi081 ceph-mon[23696]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:44.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:44 smithi081 ceph-mon[28146]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:44 smithi105 ceph-mon[24602]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:44.956 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T01:36:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:45 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:45 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:47.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:47 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/996704439' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:36:47.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:47 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/996704439' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:36:47.497 INFO:teuthology.orchestra.run.smithi081.stdout:Scheduled to redeploy mgr.y on host 'smithi081' 2024-09-17T01:36:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:47 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/996704439' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:36:48.158 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps --refresh' 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: Schedule redeploy daemon mgr.y 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.766 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[28146]: Deploying daemon mgr.y on smithi081 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: Schedule redeploy daemon mgr.y 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.767 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.768 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:48 smithi081 ceph-mon[23696]: Deploying daemon mgr.y on smithi081 2024-09-17T01:36:48.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: Schedule redeploy daemon mgr.y 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:36:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:48 smithi105 ceph-mon[24602]: Deploying daemon mgr.y on smithi081 2024-09-17T01:36:49.791 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:49.791 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:50.493 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:36:50.493 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (2m) 5s ago 10m 26.0M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:36:50.493 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (2m) 5s ago 10m 44.0M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:36:50.493 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (2m) 5s ago 9m 55.3M - 3.5 e1d6a67b021e b48ad519c20a 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (3m) 5s ago 15m 502M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:9283 running (16m) 5s ago 16m 410M - 17.2.0 e1d6a67b021e a454a71b7b66 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (16m) 5s ago 17m 57.5M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (15m) 5s ago 15m 50.1M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (15m) 5s ago 15m 46.2M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:36:50.494 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (2m) 5s ago 11m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (2m) 5s ago 11m 7852k - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (14m) 5s ago 14m 51.7M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (14m) 5s ago 14m 53.0M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (14m) 5s ago 14m 49.0M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (13m) 5s ago 13m 52.1M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (13m) 5s ago 13m 51.4M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (13m) 5s ago 12m 49.8M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (12m) 5s ago 12m 50.3M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (12m) 5s ago 12m 50.8M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (2m) 5s ago 11m 57.0M - 2.43.0 a07b618ecd1d 2be3e3d2ea19 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (10m) 5s ago 10m 74.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:36:50.495 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (9m) 5s ago 9m 75.8M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:36:50.817 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:50 smithi081 ceph-mon[28146]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:50.818 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:50 smithi081 ceph-mon[23696]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:50.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:50 smithi105 ceph-mon[24602]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:51.141 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:36:51.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:51 smithi105 ceph-mon[24602]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:51.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:51 smithi081 ceph-mon[23696]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:51.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:51 smithi081 ceph-mon[28146]: from='client.25045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:36:52.641 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:52 smithi081 ceph-mon[23696]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:52.641 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:52 smithi081 ceph-mon[28146]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:52 smithi105 ceph-mon[24602]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:36:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:36:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:36:53] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-17T01:36:53.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:54 smithi105 ceph-mon[24602]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:54 smithi081 ceph-mon[23696]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:54.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:54 smithi081 ceph-mon[28146]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:56 smithi081 ceph-mon[23696]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:36:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:56 smithi081 ceph-mon[28146]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:36:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:56 smithi105 ceph-mon[24602]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:36:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:36:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:58 smithi105 ceph-mon[24602]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:58.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:58 smithi081 ceph-mon[23696]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:58.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:58 smithi081 ceph-mon[28146]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:36:59.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:36:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:36:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:36:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:36:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:36:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:36:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:36:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:00 smithi105 ceph-mon[24602]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:00 smithi081 ceph-mon[23696]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:00 smithi081 ceph-mon[28146]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:02 smithi105 ceph-mon[24602]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:02 smithi081 ceph-mon[23696]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:02 smithi081 ceph-mon[28146]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-09-17T01:37:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:03] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-17T01:37:03.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:04 smithi105 ceph-mon[24602]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:37:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:04 smithi081 ceph-mon[23696]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:37:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:04 smithi081 ceph-mon[28146]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T01:37:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:06 smithi081 ceph-mon[23696]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:37:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:06 smithi081 ceph-mon[28146]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:37:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:06 smithi105 ceph-mon[24602]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:37:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:08 smithi081 ceph-mon[23696]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:08.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:08 smithi081 ceph-mon[28146]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:08 smithi105 ceph-mon[24602]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:09.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:10 smithi105 ceph-mon[24602]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:10 smithi081 ceph-mon[23696]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:10.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:10 smithi081 ceph-mon[28146]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:12 smithi105 ceph-mon[24602]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:12 smithi081 ceph-mon[23696]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:12 smithi081 ceph-mon[28146]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:13.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:13] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:37:13.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:14 smithi105 ceph-mon[24602]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:14 smithi081 ceph-mon[28146]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:14 smithi081 ceph-mon[23696]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:16 smithi081 ceph-mon[23696]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:16 smithi081 ceph-mon[28146]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:16 smithi105 ceph-mon[24602]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:18 smithi081 ceph-mon[23696]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:18 smithi081 ceph-mon[28146]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:18 smithi105 ceph-mon[24602]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:19.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:20 smithi105 ceph-mon[24602]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:20.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:20 smithi081 ceph-mon[23696]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:20 smithi081 ceph-mon[28146]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:22 smithi105 ceph-mon[24602]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:22 smithi081 ceph-mon[23696]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:22 smithi081 ceph-mon[28146]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:23.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:23] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:37:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:24 smithi105 ceph-mon[24602]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:24 smithi081 ceph-mon[23696]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:24 smithi081 ceph-mon[28146]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:26 smithi081 ceph-mon[23696]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:26.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:26 smithi081 ceph-mon[28146]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:26.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:26 smithi105 ceph-mon[24602]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:27.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:28.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:28 smithi081 ceph-mon[23696]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:28 smithi081 ceph-mon[28146]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:28 smithi105 ceph-mon[24602]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:29.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:29.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:30 smithi081 ceph-mon[23696]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:30.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:30 smithi081 ceph-mon[28146]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:30 smithi105 ceph-mon[24602]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:32 smithi105 ceph-mon[24602]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:32 smithi081 ceph-mon[23696]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:32 smithi081 ceph-mon[28146]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:33.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:33] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-17T01:37:33.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:34 smithi105 ceph-mon[24602]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:34 smithi081 ceph-mon[23696]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:34 smithi081 ceph-mon[28146]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:36 smithi081 ceph-mon[23696]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:36 smithi081 ceph-mon[28146]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:36 smithi105 ceph-mon[24602]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:38 smithi081 ceph-mon[23696]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:38.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:38 smithi081 ceph-mon[28146]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:38 smithi105 ceph-mon[24602]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:39.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:40.438 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:40 smithi081 ceph-mon[23696]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:40.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:40 smithi081 ceph-mon[28146]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:40 smithi105 ceph-mon[24602]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:41.775 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:41 smithi081 systemd[1]: Stopping Ceph mgr.y for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:37:42.090 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:41 smithi081 podman[60761]: 2024-09-17 01:37:41.777969729 +0000 UTC m=+0.143751078 container died a454a71b7b66dd6260ab8eb7caf0bf9588510e6423153354cb47001e5e61c1be (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.tags=base centos centos-stream, architecture=x86_64, name=centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux ) 2024-09-17T01:37:42.090 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 podman[60761]: 2024-09-17 01:37:42.093027743 +0000 UTC m=+0.458809088 container cleanup a454a71b7b66dd6260ab8eb7caf0bf9588510e6423153354cb47001e5e61c1be (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, ceph=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T01:37:42.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:42 smithi081 ceph-mon[23696]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:42.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:42.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:42 smithi081 ceph-mon[28146]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:42.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:42.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 bash[60761]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y 2024-09-17T01:37:42.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 podman[60775]: 2024-09-17 01:37:42.332128754 +0000 UTC m=+0.551879704 container remove a454a71b7b66dd6260ab8eb7caf0bf9588510e6423153354cb47001e5e61c1be (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public) 2024-09-17T01:37:42.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:37:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:42 smithi105 ceph-mon[24602]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:42.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-17T01:37:42.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 systemd[1]: Stopped Ceph mgr.y for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:37:42.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service: Consumed 59.677s CPU time. 2024-09-17T01:37:42.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 systemd[1]: Starting Ceph mgr.y for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:37:43.247 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 podman[60901]: 2024-09-17 01:37:42.882953771 +0000 UTC m=+0.015701812 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:37:43.247 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:42 smithi081 podman[60901]: 2024-09-17 01:37:42.9869899 +0000 UTC m=+0.119737932 container create a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T01:37:43.247 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 podman[60901]: 2024-09-17 01:37:43.22277063 +0000 UTC m=+0.355518670 container init a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:37:43.247 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 podman[60901]: 2024-09-17 01:37:43.225640254 +0000 UTC m=+0.358388285 container start a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, RELEASE=squid-c2ddf97, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T01:37:43.574 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-mgr[60921]: -- 172.21.15.81:0/3469076376 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b3837310e0 con 0x55b38370f000 2024-09-17T01:37:43.574 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 bash[60901]: a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 2024-09-17T01:37:43.574 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 systemd[1]: Started Ceph mgr.y for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:37:43.574 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.355+0000 7fa2ea399040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:37:43.574 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.420+0000 7fa2ea399040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:37:43.575 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.575+0000 7fa2ea399040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:37:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:43] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:37:43.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:43.503Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.642+0000 7fa2ea399040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:37:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.812+0000 7fa2ea399040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:37:44.279 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:43.956+0000 7fa2ea399040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:37:44.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[23696]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:44.281+0000 7fa2ea399040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:37:44.532 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:44.342+0000 7fa2ea399040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:37:44.533 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[28146]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:44.533 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.533 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.533 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.533 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:44 smithi105 ceph-mon[24602]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:44 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:44.914 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:44.533+0000 7fa2ea399040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:37:45.839 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:45.529+0000 7fa2ea399040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:37:45.839 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:45.605+0000 7fa2ea399040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:37:46.349 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:46 smithi105 ceph-mon[24602]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-mon[23696]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:46.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-mon[28146]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:46.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.359+0000 7fa2ea399040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:37:46.842 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.478+0000 7fa2ea399040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:37:46.864 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.628+0000 7fa2ea399040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:37:46.864 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.691+0000 7fa2ea399040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:37:47.142 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.844+0000 7fa2ea399040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:37:47.142 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.912+0000 7fa2ea399040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:37:47.142 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:46.978+0000 7fa2ea399040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:37:47.142 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.143+0000 7fa2ea399040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:37:47.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.210+0000 7fa2ea399040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:37:47.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.288+0000 7fa2ea399040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:37:47.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.373+0000 7fa2ea399040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:37:47.838 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.434+0000 7fa2ea399040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:37:47.838 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:47.495+0000 7fa2ea399040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:48.308+0000 7fa2ea399040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:37:48] ENGINE Bus STARTING 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: CherryPy Checker: 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: The Application mounted at '' has an empty config. 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:37:48] ENGINE Serving on http://:::9283 2024-09-17T01:37:48.414 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:37:48] ENGINE Bus STARTED 2024-09-17T01:37:48.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:48.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: Standby manager daemon y restarted 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: Standby manager daemon y started 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: Standby manager daemon y restarted 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: Standby manager daemon y started 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:48 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: Standby manager daemon y restarted 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: Standby manager daemon y started 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:48.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:48 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:49 smithi105 ceph-mon[24602]: mgrmap e27: x(active, since 4m), standbys: y 2024-09-17T01:37:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:37:49.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:49 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:37:49.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:49 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[23696]: mgrmap e27: x(active, since 4m), standbys: y 2024-09-17T01:37:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:37:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:37:49.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:49.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:49.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[28146]: mgrmap e27: x(active, since 4m), standbys: y 2024-09-17T01:37:49.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:37:49.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:37:49.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:49 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:37:50.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:50 smithi105 ceph-mon[24602]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:50.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:50 smithi081 ceph-mon[23696]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:50.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:50 smithi081 ceph-mon[28146]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:52.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:52 smithi105 ceph-mon[24602]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:52.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:52 smithi081 ceph-mon[23696]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:52.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:52 smithi081 ceph-mon[28146]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:37:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:37:53] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:37:53.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:54 smithi105 ceph-mon[24602]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:54 smithi081 ceph-mon[23696]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:54.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:54 smithi081 ceph-mon[28146]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:56 smithi081 ceph-mon[23696]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:56 smithi081 ceph-mon[28146]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:56 smithi105 ceph-mon[24602]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:37:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:37:58.366 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:58 smithi081 ceph-mon[23696]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:58.366 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:58 smithi081 ceph-mon[28146]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:58 smithi105 ceph-mon[24602]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:37:59.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:37:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:37:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:37:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:37:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:37:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:37:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:37:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:00 smithi105 ceph-mon[24602]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:00 smithi081 ceph-mon[23696]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:00 smithi081 ceph-mon[28146]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:02 smithi105 ceph-mon[24602]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:02 smithi081 ceph-mon[23696]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:02 smithi081 ceph-mon[28146]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:03] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:38:03.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:04 smithi105 ceph-mon[24602]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:04 smithi081 ceph-mon[23696]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:04 smithi081 ceph-mon[28146]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:06 smithi081 ceph-mon[23696]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:06 smithi081 ceph-mon[28146]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:06 smithi105 ceph-mon[24602]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:08 smithi081 ceph-mon[23696]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:08 smithi081 ceph-mon[28146]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:08 smithi105 ceph-mon[24602]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:09.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:10 smithi105 ceph-mon[24602]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:10 smithi081 ceph-mon[23696]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:10.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:10 smithi081 ceph-mon[28146]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:12 smithi105 ceph-mon[24602]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:12 smithi081 ceph-mon[23696]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:12 smithi081 ceph-mon[28146]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:13.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:13] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:38:13.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:14 smithi105 ceph-mon[24602]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:14 smithi081 ceph-mon[23696]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:14 smithi081 ceph-mon[28146]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:16 smithi081 ceph-mon[23696]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:16 smithi081 ceph-mon[28146]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:16 smithi105 ceph-mon[24602]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:18 smithi081 ceph-mon[23696]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:18 smithi081 ceph-mon[28146]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:18 smithi105 ceph-mon[24602]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:19.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:19.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:20 smithi105 ceph-mon[24602]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:20.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:20 smithi081 ceph-mon[23696]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:20 smithi081 ceph-mon[28146]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:22 smithi105 ceph-mon[24602]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:22 smithi081 ceph-mon[23696]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:22 smithi081 ceph-mon[28146]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:23.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:23] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:38:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:24 smithi105 ceph-mon[24602]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:24 smithi081 ceph-mon[23696]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:24 smithi081 ceph-mon[28146]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:26 smithi081 ceph-mon[23696]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:26.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:26 smithi081 ceph-mon[28146]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:26.437 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:26 smithi105 ceph-mon[24602]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:28.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:28 smithi081 ceph-mon[23696]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:28 smithi081 ceph-mon[28146]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:28 smithi105 ceph-mon[24602]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:29.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:30.469 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:30 smithi081 ceph-mon[23696]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:30.470 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:30 smithi081 ceph-mon[28146]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:30 smithi105 ceph-mon[24602]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:32 smithi105 ceph-mon[24602]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:32 smithi081 ceph-mon[23696]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:32 smithi081 ceph-mon[28146]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:33.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:33] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-17T01:38:33.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:34 smithi105 ceph-mon[24602]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:34 smithi081 ceph-mon[23696]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:34 smithi081 ceph-mon[28146]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:36 smithi105 ceph-mon[24602]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:36 smithi081 ceph-mon[23696]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:36 smithi081 ceph-mon[28146]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:38 smithi081 ceph-mon[23696]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:38 smithi081 ceph-mon[28146]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:38 smithi105 ceph-mon[24602]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:39.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:40 smithi105 ceph-mon[24602]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:40.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:40 smithi081 ceph-mon[23696]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:40 smithi081 ceph-mon[28146]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:42 smithi105 ceph-mon[24602]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:42 smithi081 ceph-mon[23696]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:42 smithi081 ceph-mon[28146]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:43] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.43.0" 2024-09-17T01:38:43.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:44 smithi105 ceph-mon[24602]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:44 smithi081 ceph-mon[23696]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:44 smithi081 ceph-mon[28146]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:46 smithi105 ceph-mon[24602]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:46 smithi081 ceph-mon[23696]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:46 smithi081 ceph-mon[28146]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:48 smithi081 ceph-mon[23696]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:48.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:48 smithi081 ceph-mon[28146]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:48 smithi105 ceph-mon[24602]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:49.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:50.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:50 smithi105 ceph-mon[24602]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:50.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:50 smithi081 ceph-mon[23696]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:50.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:50 smithi081 ceph-mon[28146]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:52.540 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:52 smithi081 ceph-mon[23696]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:52.541 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:52 smithi081 ceph-mon[28146]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:52 smithi105 ceph-mon[24602]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:38:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:38:53] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.43.0" 2024-09-17T01:38:53.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:38:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:38:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:54 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:38:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:54 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:38:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:56 smithi105 ceph-mon[24602]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:56 smithi081 ceph-mon[23696]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:56 smithi081 ceph-mon[28146]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:38:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:38:58.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:58 smithi081 ceph-mon[23696]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:58.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:58 smithi081 ceph-mon[28146]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:58 smithi105 ceph-mon[24602]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:38:59.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:38:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:38:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:38:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:38:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:38:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:38:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:38:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:00 smithi105 ceph-mon[24602]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:00.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:00 smithi081 ceph-mon[23696]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:00.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:00 smithi081 ceph-mon[28146]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:02 smithi105 ceph-mon[24602]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:02 smithi081 ceph-mon[23696]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:02 smithi081 ceph-mon[28146]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:03.503 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:03] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-17T01:39:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:04 smithi105 ceph-mon[24602]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:04.636 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:39:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:39:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi081\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi081\", job=\"node\", machine=\"x86_64\", nodename=\"smithi081\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:39:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:04 smithi081 ceph-mon[23696]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:04 smithi081 ceph-mon[28146]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:06 smithi105 ceph-mon[24602]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:06 smithi081 ceph-mon[23696]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:06 smithi081 ceph-mon[28146]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:08 smithi105 ceph-mon[24602]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:08 smithi081 ceph-mon[23696]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:08 smithi081 ceph-mon[28146]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:09.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:39:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:39:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"172.21.15.105:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.81\", device_class=\"ssd\", hostname=\"smithi081\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.81\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:39:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:10 smithi105 ceph-mon[24602]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:10 smithi081 ceph-mon[23696]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:10.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:10 smithi081 ceph-mon[28146]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:12 smithi105 ceph-mon[24602]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:12 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:12 smithi081 ceph-mon[23696]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:12 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:12 smithi081 ceph-mon[28146]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:12 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:13.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:13] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:39:13.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:39:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:39:13.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ded6f264-7492-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-17T01:39:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:14 smithi105 ceph-mon[24602]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:14 smithi081 ceph-mon[23696]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:14 smithi081 ceph-mon[28146]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:16 smithi105 ceph-mon[24602]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:16 smithi081 ceph-mon[23696]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:16 smithi081 ceph-mon[28146]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:18.483 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:18 smithi081 ceph-mon[23696]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:18.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:18 smithi081 ceph-mon[28146]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:18 smithi105 ceph-mon[24602]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:19 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:19 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:19 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:20 smithi105 ceph-mon[24602]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:20.669 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:20 smithi081 ceph-mon[23696]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:20.669 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:20 smithi081 ceph-mon[28146]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:22 smithi105 ceph-mon[24602]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:22 smithi081 ceph-mon[23696]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:22 smithi081 ceph-mon[28146]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:23.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:23] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.43.0" 2024-09-17T01:39:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:24 smithi105 ceph-mon[24602]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:24 smithi081 ceph-mon[23696]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:24 smithi081 ceph-mon[28146]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:26 smithi105 ceph-mon[24602]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:26 smithi081 ceph-mon[23696]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:26 smithi081 ceph-mon[28146]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:27 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:27 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:27 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:28.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:28 smithi105 ceph-mon[24602]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:28.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:28 smithi081 ceph-mon[28146]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:28 smithi081 ceph-mon[23696]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:29 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:29 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:29 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:30 smithi105 ceph-mon[24602]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:30.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:30 smithi081 ceph-mon[23696]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:30.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:30 smithi081 ceph-mon[28146]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:32 smithi105 ceph-mon[24602]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:32 smithi081 ceph-mon[23696]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:32 smithi081 ceph-mon[28146]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:33.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:33] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.43.0" 2024-09-17T01:39:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:34 smithi105 ceph-mon[24602]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:34 smithi081 ceph-mon[23696]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:34 smithi081 ceph-mon[28146]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:36 smithi105 ceph-mon[24602]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:36 smithi081 ceph-mon[23696]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:36 smithi081 ceph-mon[28146]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:38 smithi105 ceph-mon[24602]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:38 smithi081 ceph-mon[23696]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:38.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:38 smithi081 ceph-mon[28146]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:39 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:39 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:39 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:40.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:40 smithi105 ceph-mon[24602]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:40.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:40 smithi081 ceph-mon[23696]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:40.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:40 smithi081 ceph-mon[28146]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:42 smithi105 ceph-mon[24602]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:42 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:42 smithi081 ceph-mon[23696]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:42 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:42 smithi081 ceph-mon[28146]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:42.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:42 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:43.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:43] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-17T01:39:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:44 smithi105 ceph-mon[24602]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:44 smithi081 ceph-mon[23696]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:44 smithi081 ceph-mon[28146]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:46 smithi105 ceph-mon[24602]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:46 smithi081 ceph-mon[23696]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:46 smithi081 ceph-mon[28146]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:48.484 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:48 smithi081 ceph-mon[23696]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:48.484 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:48 smithi081 ceph-mon[28146]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:48 smithi105 ceph-mon[24602]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:49 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:49 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:49 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:50.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:50 smithi105 ceph-mon[24602]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:50.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:50 smithi081 ceph-mon[23696]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:50.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:50 smithi081 ceph-mon[28146]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:52 smithi105 ceph-mon[24602]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:52 smithi081 ceph-mon[23696]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:52.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:52 smithi081 ceph-mon[28146]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:39:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:39:53] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.43.0" 2024-09-17T01:39:54.429 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:54 smithi081 ceph-mon[23696]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:54.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:54 smithi081 ceph-mon[28146]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:54.456 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:39:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:54 smithi105 ceph-mon[24602]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[23696]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:39:55.421 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:55 smithi081 ceph-mon[28146]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:39:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:55 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:39:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:55 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:39:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:55 smithi105 ceph-mon[24602]: from='mgr.34721 ' entity='mgr.x' 2024-09-17T01:39:56.427 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:39:56.427 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (5m) 63s ago 13m 24.9M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (5m) 2m ago 13m 45.7M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (5m) 63s ago 12m 56.0M - 3.5 e1d6a67b021e b48ad519c20a 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (6m) 2m ago 18m 505M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (2m) 63s ago 19m 438M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (20m) 63s ago 20m 58.7M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (18m) 2m ago 18m 47.6M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (18m) 63s ago 18m 44.1M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (6m) 63s ago 14m 15.2M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (5m) 2m ago 14m 15.7M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:39:56.428 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (17m) 63s ago 17m 52.8M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (17m) 63s ago 17m 54.1M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (17m) 63s ago 17m 49.7M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (16m) 63s ago 16m 53.2M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (16m) 2m ago 16m 52.4M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (16m) 2m ago 16m 51.5M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (15m) 2m ago 15m 50.4M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (15m) 2m ago 15m 52.0M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (5m) 2m ago 14m 59.1M - 2.43.0 a07b618ecd1d 2be3e3d2ea19 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (13m) 63s ago 13m 75.1M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:39:56.429 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (13m) 2m ago 13m 76.0M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:39:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:56 smithi105 ceph-mon[24602]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:56 smithi081 ceph-mon[23696]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:56 smithi081 ceph-mon[28146]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:39:57.002 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:39:57.631 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:57 smithi081 ceph-mon[23696]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:39:57.632 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:57 smithi081 ceph-mon[23696]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:57.632 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:57 smithi081 ceph-mon[28146]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:39:57.632 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:57 smithi081 ceph-mon[28146]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:57 smithi105 ceph-mon[24602]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:39:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:57 smithi105 ceph-mon[24602]: from='mgr.34721 172.21.15.105:0/1349217411' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:39:58.564 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:58 smithi081 ceph-mon[23696]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:58.564 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:58 smithi081 ceph-mon[28146]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:58 smithi105 ceph-mon[24602]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:39:59.059 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:39:59.059 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:39:59.059 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:39:59.060 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:39:59.061 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:39:59.061 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:39:59.061 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:39:59.061 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:39:59.061 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:39:59.618 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:59 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:59.618 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:39:59 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4106923172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:39:59.618 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:59 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:59.619 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:39:59 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4106923172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:39:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:59 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:39:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:39:59 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/4106923172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:39:59.640 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:40:00.574 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:00 smithi081 ceph-mon[23696]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:00.575 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:00 smithi081 ceph-mon[23696]: overall HEALTH_OK 2024-09-17T01:40:00.575 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:00 smithi081 ceph-mon[28146]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:00.575 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:00 smithi081 ceph-mon[28146]: overall HEALTH_OK 2024-09-17T01:40:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:00 smithi105 ceph-mon[24602]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:00 smithi105 ceph-mon[24602]: overall HEALTH_OK 2024-09-17T01:40:01.642 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_OK 2024-09-17T01:40:02.230 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph -s' 2024-09-17T01:40:02.528 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:02 smithi081 ceph-mon[28146]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:02.528 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:02 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4142092484' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:40:02.528 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:02 smithi081 ceph-mon[23696]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:02.529 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:02 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4142092484' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:40:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:02 smithi105 ceph-mon[24602]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:02 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/4142092484' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:40:03.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:40:03] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.43.0" 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: cluster: 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: id: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: health: HEALTH_OK 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: services: 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: mgr: x(active, since 6m), standbys: y 2024-09-17T01:40:04.302 INFO:teuthology.orchestra.run.smithi081.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: data: 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: pools: 6 pools, 161 pgs 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: objects: 241 objects, 585 KiB 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: pgs: 161 active+clean 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: io: 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-09-17T01:40:04.303 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:40:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:04 smithi105 ceph-mon[24602]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:04 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3351616873' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:40:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:04 smithi081 ceph-mon[23696]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:04.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:04 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3351616873' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:40:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:04 smithi081 ceph-mon[28146]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:04 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3351616873' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:40:04.987 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph mgr fail' 2024-09-17T01:40:06.344 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:06 smithi081 ceph-mon[23696]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:06.345 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:06 smithi081 ceph-mon[28146]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:06 smithi105 ceph-mon[24602]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:07.554 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:40:07.555 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-mon[23696]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T01:40:07.555 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE Bus STOPPING 2024-09-17T01:40:07.555 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:40:07.555 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-mon[28146]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T01:40:07.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-mon[24602]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.223+0000 7f64ef2f3640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ignoring --setuser ceph since I am not root 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ignoring --setgroup ceph since I am not root 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-mgr[44255]: -- 172.21.15.105:0/3510257723 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559fb491d0e0 con 0x559fb48fb000 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.374+0000 7f02c0dbc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.429+0000 7f02c0dbc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:40:07.574 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.575+0000 7f02c0dbc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:40:07.836 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:40:07.836 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE Bus STOPPED 2024-09-17T01:40:07.836 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE Bus STARTING 2024-09-17T01:40:07.836 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE Serving on http://:::9283 2024-09-17T01:40:07.836 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:07 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:40:07] ENGINE Bus STARTED 2024-09-17T01:40:07.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.642+0000 7f02c0dbc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:40:07.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.791+0000 7f02c0dbc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:40:07.903 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:40:08.226 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:07.935+0000 7f02c0dbc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:40:08.460 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:40:08.460 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: mgrmap e28: y(active, starting, since 0.280052s) 2024-09-17T01:40:08.460 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:40:08.460 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:40:08.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: mgrmap e28: y(active, starting, since 0.280052s) 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:40:08.462 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: Manager daemon y is now available 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:40:08.463 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:40:08.539 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:08.262+0000 7f02c0dbc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:40:08.539 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:08.336+0000 7f02c0dbc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:40:08.539 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:08.541+0000 7f02c0dbc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:40:08.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3646626051' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: mgrmap e28: y(active, starting, since 0.280052s) 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:40:08.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: Manager daemon y is now available 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:40:08.541 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:40:08.879+0000 7fa246b93640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:01:40:08] ENGINE Error in HTTPServer.serve 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._connections.run(self.expiration_interval) 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._run(expiration_interval) 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:40:09.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s = self.context.wrap_socket( 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.sslsocket_class._create( 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self.do_handshake() 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._sslobj.do_handshake() 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:40:09.182 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:40:09.478 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[23696]: mgrmap e29: y(active, since 1.28572s) 2024-09-17T01:40:09.478 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[23696]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:09.478 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[23696]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:09.479 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[28146]: mgrmap e29: y(active, since 1.28572s) 2024-09-17T01:40:09.479 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[28146]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:09.479 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:09 smithi081 ceph-mon[28146]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:09.529 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:09.468+0000 7f02c0dbc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:40:09.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:09 smithi105 ceph-mon[24602]: mgrmap e29: y(active, since 1.28572s) 2024-09-17T01:40:09.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:09 smithi105 ceph-mon[24602]: from='client.24976 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:09.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:09 smithi105 ceph-mon[24602]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:09.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:09.541+0000 7f02c0dbc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:40:10.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.285+0000 7f02c0dbc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:40:10.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.407+0000 7f02c0dbc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: [17/Sep/2024:01:40:08] ENGINE Bus STARTING 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: [17/Sep/2024:01:40:08] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: [17/Sep/2024:01:40:08] ENGINE Error in HTTPServer.serve 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:40:10.529 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: [17/Sep/2024:01:40:08] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:40:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: [17/Sep/2024:01:40:08] ENGINE Bus STARTED 2024-09-17T01:40:10.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:10.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-mon[24602]: mgrmap e30: y(active, since 2s) 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: [17/Sep/2024:01:40:08] ENGINE Bus STARTING 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: [17/Sep/2024:01:40:08] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: [17/Sep/2024:01:40:08] ENGINE Error in HTTPServer.serve 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:40:10.623 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:40:10.624 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.do_handshake() 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: [17/Sep/2024:01:40:08] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: [17/Sep/2024:01:40:08] ENGINE Bus STARTED 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[28146]: mgrmap e30: y(active, since 2s) 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: [17/Sep/2024:01:40:08] ENGINE Bus STARTING 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: [17/Sep/2024:01:40:08] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:40:10.625 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: [17/Sep/2024:01:40:08] ENGINE Error in HTTPServer.serve 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:40:10.626 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.do_handshake() 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: [17/Sep/2024:01:40:08] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: [17/Sep/2024:01:40:08] ENGINE Bus STARTED 2024-09-17T01:40:10.627 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:10.628 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:10 smithi081 ceph-mon[23696]: mgrmap e30: y(active, since 2s) 2024-09-17T01:40:10.849 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.556+0000 7f02c0dbc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:40:10.850 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.617+0000 7f02c0dbc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:40:10.850 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.776+0000 7f02c0dbc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:40:10.850 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.850+0000 7f02c0dbc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:40:11.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:10.910+0000 7f02c0dbc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:40:11.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.075+0000 7f02c0dbc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:40:11.416 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.141+0000 7f02c0dbc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:40:11.416 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.209+0000 7f02c0dbc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:40:11.416 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.297+0000 7f02c0dbc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:40:11.416 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.357+0000 7f02c0dbc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:40:11.417 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:11.416+0000 7f02c0dbc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: mgrmap e31: y(active, since 4s) 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:40:12.455 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: mgrmap e31: y(active, since 4s) 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:40:12.456 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:40:12.457 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:12 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:40:12.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:12.235+0000 7f02c0dbc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:40:12.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:40:12] ENGINE Bus STARTING 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: CherryPy Checker: 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: The Application mounted at '' has an empty config. 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:40:12] ENGINE Serving on http://:::9283 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:40:12] ENGINE Bus STARTED 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: mgrmap e31: y(active, since 4s) 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:40:12.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:40:12.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:40:12.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:40:12.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:12 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:40:13.822 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-17T01:40:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[23696]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[23696]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-17T01:40:13.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:40:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[28146]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[28146]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-17T01:40:13.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:13 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:40:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-mon[24602]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-mon[24602]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-17T01:40:14.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:13 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:40:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:15 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:15.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:15 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:16.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:40:16.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:16.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:16.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:16 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:40:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:16.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:16 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:17 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:17 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:17 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:40:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:17 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:17 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:18.539 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-mon[23696]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:40:18.540 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-mon[28146]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:40:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:18 smithi105 ceph-mon[24602]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:40:18.656+0000 7fa27a5f7640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Creating ceph-iscsi config... 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return _run_code(code, main_globals, None, 2024-09-17T01:40:18.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: exec(code, run_globals) 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:18.932 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.wait_async( 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return future.result(timeout) 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:40:18.933 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.__get_result() 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: raise self._exception 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: out, err, code = await self._run_cephadm( 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: raise OrchestratorError( 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Creating ceph-iscsi config... 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:18.934 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return _run_code(code, main_globals, None, 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: exec(code, run_globals) 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:18.935 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:18 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:19 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.593 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.594 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:40:19.595 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.596 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:19 smithi105 ceph-mon[24602]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:19 smithi105 ceph-mon[24602]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:40:19.597 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:19 smithi105 ceph-mon[24602]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:40:19.665 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.665 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.666 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:40:19.667 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.668 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.669 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:40:19.670 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.671 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.672 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[23696]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[23696]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[23696]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:40:19.673 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:40:19.674 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[28146]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[28146]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:40:19.675 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:19 smithi081 ceph-mon[28146]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:40:21.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:20.728Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.105:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.105:8765: connect: connection refused" 2024-09-17T01:40:21.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:20.729Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.105:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.105:8765: connect: connection refused" 2024-09-17T01:40:21.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:20.730Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.105:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.105:8765: connect: connection refused" 2024-09-17T01:40:21.024 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:20.732Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.105:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.105:8765: connect: connection refused" 2024-09-17T01:40:21.024 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:20 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:20.732Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.105:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.105:8765: connect: connection refused" 2024-09-17T01:40:21.589 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:21 smithi105 ceph-mon[24602]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:21 smithi081 ceph-mon[23696]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:21.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:21 smithi081 ceph-mon[28146]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:22.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.262Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.263Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.263Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-17T01:40:22.553 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[47548]: ts=2024-09-17T01:40:22.263Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-17T01:40:22.554 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 podman[53036]: 2024-09-17 01:40:22.272938584 +0000 UTC m=+0.111691970 container died 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:22.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:22 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:22.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 podman[53036]: 2024-09-17 01:40:22.670842498 +0000 UTC m=+0.509595884 container cleanup 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:22.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 bash[53036]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T01:40:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:22 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:22 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:23.238 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:22 smithi105 podman[53048]: 2024-09-17 01:40:22.896955522 +0000 UTC m=+0.626344734 container remove 2be3e3d2ea1942f30bff12c292917d354c65a08ce7fdcdc4726534b104eda8d1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:23.238 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T01:40:23.238 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 systemd[1]: Stopped Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:40:23.238 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.343s CPU time. 2024-09-17T01:40:23.239 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 systemd[1]: Starting Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:40:23.514 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 podman[53143]: 2024-09-17 01:40:23.144325308 +0000 UTC m=+0.016741282 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-17T01:40:23.514 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 podman[53143]: 2024-09-17 01:40:23.282819548 +0000 UTC m=+0.155235506 container create 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-mon[24602]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:23.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:23.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 podman[53143]: 2024-09-17 01:40:23.516976197 +0000 UTC m=+0.389392165 container init 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 podman[53143]: 2024-09-17 01:40:23.519883218 +0000 UTC m=+0.392299178 container start 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.543Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.543Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.543Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.543Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.543Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T01:40:23.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.544Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.545Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.545Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.545Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.548Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.548Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.797µs 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.548Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.555Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.570Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.579Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.580Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.580Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.995µs wal_replay_duration=31.711039ms wbl_replay_duration=88ns total_replay_duration=31.750392ms 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.583Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T01:40:23.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.583Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.583Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 bash[53143]: 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.604Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.222111ms db_storage=859ns remote_storage=1.155µs web_handler=700ns query_engine=631ns scrape=477.979µs scrape_sd=94.612µs notify=18.978µs notify_sd=26.626µs rules=20.266071ms tracing=6.694µs 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.604Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-17T01:40:23.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:40:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:40:23.604Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-17T01:40:24.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[23696]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:24.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:24.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:24.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:24.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[28146]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:24.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:24.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:24.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:23 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:24.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:24 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:25.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:24 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:25.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:24 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:40:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:26 smithi105 ceph-mon[24602]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:26 smithi081 ceph-mon[23696]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:26 smithi081 ceph-mon[28146]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:40:27.594 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:27 smithi105 ceph-mon[24602]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:27 smithi081 ceph-mon[23696]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:27 smithi081 ceph-mon[28146]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:28.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:28 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:29.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:28 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:40:29.996 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:29 smithi081 ceph-mon[23696]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:29.996 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:29 smithi081 ceph-mon[28146]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:30.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:29 smithi105 ceph-mon[24602]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:40:31.621 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:31 smithi081 ceph-mon[23696]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:31.621 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:31 smithi081 ceph-mon[28146]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:31 smithi105 ceph-mon[24602]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1969422994' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[28146]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:40:33] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.43.0" 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1969422994' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:40:33.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:33 smithi081 ceph-mon[23696]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:33.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:33.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T01:40:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T01:40:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1969422994' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:40:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:33 smithi105 ceph-mon[24602]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:40:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:36 smithi081 ceph-mon[23696]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:36 smithi081 ceph-mon[28146]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:36 smithi105 ceph-mon[24602]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:38.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:38.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:38.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:38.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:40:38.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:39 smithi105 ceph-mon[24602]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:39 smithi081 ceph-mon[23696]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:39 smithi081 ceph-mon[28146]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:41 smithi105 ceph-mon[24602]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:41 smithi081 ceph-mon[23696]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:41 smithi081 ceph-mon[28146]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:43 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:43 smithi105 ceph-mon[24602]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:43 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:43 smithi081 ceph-mon[23696]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:43 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:43 smithi081 ceph-mon[28146]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:40:43.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:40:43] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.43.0" 2024-09-17T01:40:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:46 smithi105 ceph-mon[24602]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-17T01:40:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:46 smithi081 ceph-mon[23696]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-17T01:40:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:46 smithi081 ceph-mon[28146]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-09-17T01:40:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:47 smithi105 ceph-mon[24602]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:47 smithi081 ceph-mon[23696]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:47 smithi081 ceph-mon[28146]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:49 smithi105 ceph-mon[24602]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:49 smithi081 ceph-mon[23696]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:49 smithi081 ceph-mon[28146]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:51 smithi105 ceph-mon[24602]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:51 smithi081 ceph-mon[23696]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:51 smithi081 ceph-mon[28146]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:52 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:52 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:53.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:52 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:40:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:53 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:53 smithi081 ceph-mon[23696]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:53 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:53 smithi081 ceph-mon[28146]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:53.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:40:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:40:53] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-09-17T01:40:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:53 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:40:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:53 smithi105 ceph-mon[24602]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:56 smithi105 ceph-mon[24602]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:56 smithi081 ceph-mon[23696]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:56 smithi081 ceph-mon[28146]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:57 smithi105 ceph-mon[24602]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:57 smithi081 ceph-mon[23696]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:57 smithi081 ceph-mon[28146]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:40:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:40:59 smithi105 ceph-mon[24602]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:40:59 smithi081 ceph-mon[23696]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:40:59.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:40:59 smithi081 ceph-mon[28146]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:01 smithi105 ceph-mon[24602]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:01 smithi081 ceph-mon[23696]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:01 smithi081 ceph-mon[28146]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:03.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:03 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:03.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:03 smithi105 ceph-mon[24602]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:03.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:03] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-09-17T01:41:03.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:03 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:03.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:03 smithi081 ceph-mon[23696]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:03.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:03 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:03.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:03 smithi081 ceph-mon[28146]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:06 smithi105 ceph-mon[24602]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:06 smithi081 ceph-mon[23696]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:06 smithi081 ceph-mon[28146]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:07 smithi105 ceph-mon[24602]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:07.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:07 smithi081 ceph-mon[23696]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:07.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:07 smithi081 ceph-mon[28146]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:09 smithi105 ceph-mon[24602]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:09 smithi081 ceph-mon[23696]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:09.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:09 smithi081 ceph-mon[28146]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:11 smithi105 ceph-mon[24602]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:11 smithi081 ceph-mon[23696]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:11 smithi081 ceph-mon[28146]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:13 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:13 smithi105 ceph-mon[24602]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:13.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:13 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:13.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:13 smithi081 ceph-mon[28146]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:13 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:13 smithi081 ceph-mon[23696]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:13.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:13] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-09-17T01:41:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:16 smithi105 ceph-mon[24602]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:16 smithi081 ceph-mon[28146]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:16 smithi081 ceph-mon[23696]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:17 smithi105 ceph-mon[24602]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:17.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:17 smithi081 ceph-mon[28146]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:17.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:17 smithi081 ceph-mon[23696]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:19 smithi105 ceph-mon[24602]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:19 smithi081 ceph-mon[23696]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:19 smithi081 ceph-mon[28146]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:21 smithi105 ceph-mon[24602]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:21 smithi081 ceph-mon[23696]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:21 smithi081 ceph-mon[28146]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:22 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:22 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:22 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:23 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:23 smithi081 ceph-mon[23696]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:23] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:41:23.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:23 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:23.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:23 smithi081 ceph-mon[28146]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:23 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:23 smithi105 ceph-mon[24602]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:26 smithi105 ceph-mon[24602]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:26 smithi081 ceph-mon[23696]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:26 smithi081 ceph-mon[28146]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:27 smithi105 ceph-mon[24602]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:27 smithi081 ceph-mon[23696]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:27 smithi081 ceph-mon[28146]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:29 smithi105 ceph-mon[24602]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:29 smithi081 ceph-mon[23696]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:29 smithi081 ceph-mon[28146]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:31 smithi105 ceph-mon[24602]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:31 smithi081 ceph-mon[23696]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:31 smithi081 ceph-mon[28146]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:33 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:33 smithi105 ceph-mon[24602]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:33 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:33 smithi081 ceph-mon[28146]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:33 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:33 smithi081 ceph-mon[23696]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:33.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:33] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:41:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:36 smithi105 ceph-mon[24602]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:36 smithi081 ceph-mon[23696]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:36 smithi081 ceph-mon[28146]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:37 smithi105 ceph-mon[24602]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:37 smithi081 ceph-mon[23696]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:37.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:37 smithi081 ceph-mon[28146]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:41:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:41:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:41:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:41:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:41:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:39 smithi105 ceph-mon[24602]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:39 smithi081 ceph-mon[23696]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:39 smithi081 ceph-mon[28146]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:41 smithi105 ceph-mon[24602]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:41 smithi081 ceph-mon[23696]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:41 smithi081 ceph-mon[28146]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:43 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:43 smithi105 ceph-mon[24602]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:43] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.43.0" 2024-09-17T01:41:43.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:43 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:43.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:43 smithi081 ceph-mon[23696]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:43 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:43 smithi081 ceph-mon[28146]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:46 smithi105 ceph-mon[24602]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:46 smithi081 ceph-mon[23696]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:46 smithi081 ceph-mon[28146]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:47 smithi105 ceph-mon[24602]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:47 smithi081 ceph-mon[23696]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:47 smithi081 ceph-mon[28146]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:49 smithi105 ceph-mon[24602]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:49.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:49 smithi081 ceph-mon[23696]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:49 smithi081 ceph-mon[28146]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:51 smithi105 ceph-mon[24602]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:51 smithi081 ceph-mon[23696]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:51 smithi081 ceph-mon[28146]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:52 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:52 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:53.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:52 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:41:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:53 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:53 smithi081 ceph-mon[23696]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:53.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:41:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:41:53] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-17T01:41:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:53 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:53 smithi081 ceph-mon[28146]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:53 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:41:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:53 smithi105 ceph-mon[24602]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:56 smithi105 ceph-mon[24602]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:56 smithi081 ceph-mon[23696]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:56 smithi081 ceph-mon[28146]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:57 smithi105 ceph-mon[24602]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:57 smithi081 ceph-mon[23696]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:57 smithi081 ceph-mon[28146]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:41:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:41:59 smithi105 ceph-mon[24602]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:41:59 smithi081 ceph-mon[23696]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:41:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:41:59 smithi081 ceph-mon[28146]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:01 smithi105 ceph-mon[24602]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:01 smithi081 ceph-mon[23696]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:01 smithi081 ceph-mon[28146]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:03 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:03 smithi105 ceph-mon[24602]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:03.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:03 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:03.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:03 smithi081 ceph-mon[28146]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:03 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:03.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:03 smithi081 ceph-mon[23696]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:03.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:03] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-09-17T01:42:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:06 smithi105 ceph-mon[24602]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:06 smithi081 ceph-mon[23696]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:06 smithi081 ceph-mon[28146]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:07 smithi105 ceph-mon[24602]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:07.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:07 smithi081 ceph-mon[23696]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:07.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:07 smithi081 ceph-mon[28146]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:09 smithi105 ceph-mon[24602]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:09.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:09 smithi081 ceph-mon[23696]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:09.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:09 smithi081 ceph-mon[28146]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:11 smithi105 ceph-mon[24602]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:11 smithi081 ceph-mon[23696]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:11 smithi081 ceph-mon[28146]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:13 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:13 smithi105 ceph-mon[24602]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:13 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:13 smithi081 ceph-mon[23696]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:13.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:13] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.43.0" 2024-09-17T01:42:13.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:13 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:13.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:13 smithi081 ceph-mon[28146]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:16 smithi105 ceph-mon[24602]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:16 smithi081 ceph-mon[23696]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:16 smithi081 ceph-mon[28146]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:17 smithi105 ceph-mon[24602]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:17.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:17 smithi081 ceph-mon[23696]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:17.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:17 smithi081 ceph-mon[28146]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:19 smithi105 ceph-mon[24602]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:19.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:19 smithi081 ceph-mon[23696]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:19.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:19 smithi081 ceph-mon[28146]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:21 smithi105 ceph-mon[24602]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:21 smithi081 ceph-mon[23696]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:21 smithi081 ceph-mon[28146]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:22 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:22 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:22 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:23 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:23.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:23 smithi081 ceph-mon[23696]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:23.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:23] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:42:23.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:23 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:23.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:23 smithi081 ceph-mon[28146]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:23 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:23 smithi105 ceph-mon[24602]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:26 smithi105 ceph-mon[24602]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:26 smithi081 ceph-mon[23696]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:26 smithi081 ceph-mon[28146]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:27 smithi105 ceph-mon[24602]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:27 smithi081 ceph-mon[23696]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:27.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:27 smithi081 ceph-mon[28146]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:29 smithi105 ceph-mon[24602]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:29 smithi081 ceph-mon[23696]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:29.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:29 smithi081 ceph-mon[28146]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:31 smithi105 ceph-mon[24602]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:31 smithi081 ceph-mon[23696]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:31 smithi081 ceph-mon[28146]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:33 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:33 smithi105 ceph-mon[24602]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:33 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:33 smithi081 ceph-mon[28146]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:33 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:33 smithi081 ceph-mon[23696]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:33.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:33] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:42:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:36 smithi105 ceph-mon[24602]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:36 smithi081 ceph-mon[23696]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:36 smithi081 ceph-mon[28146]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:37 smithi105 ceph-mon[24602]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:37 smithi081 ceph-mon[23696]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:37 smithi081 ceph-mon[28146]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:42:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:42:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:38 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:42:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:38.703 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:42:38.703 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:42:38.703 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:42:38.704 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:38.704 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:42:38.704 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:42:38.704 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:38 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' 2024-09-17T01:42:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:39 smithi105 ceph-mon[24602]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:39 smithi081 ceph-mon[23696]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:39 smithi081 ceph-mon[28146]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:41 smithi105 ceph-mon[24602]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:41 smithi081 ceph-mon[23696]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:41 smithi081 ceph-mon[28146]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:43 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:43 smithi105 ceph-mon[24602]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:43.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:43 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:43.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:43 smithi081 ceph-mon[28146]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:43.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:43 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:43.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:43 smithi081 ceph-mon[23696]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:43.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:43] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-09-17T01:42:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:46 smithi105 ceph-mon[24602]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:46 smithi081 ceph-mon[23696]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:46 smithi081 ceph-mon[28146]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:47 smithi105 ceph-mon[24602]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:47 smithi081 ceph-mon[23696]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:47 smithi081 ceph-mon[28146]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:49 smithi105 ceph-mon[24602]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:49.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:49 smithi081 ceph-mon[28146]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:49.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:49 smithi081 ceph-mon[23696]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:51 smithi105 ceph-mon[24602]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:51 smithi081 ceph-mon[23696]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:51 smithi081 ceph-mon[28146]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:52 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:52 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:53.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:52 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:42:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:53 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:53.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:53 smithi081 ceph-mon[23696]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:53 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:53 smithi081 ceph-mon[28146]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:53.931 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:42:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:42:53] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:42:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:53 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:42:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:53 smithi105 ceph-mon[24602]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:56 smithi105 ceph-mon[24602]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:56 smithi081 ceph-mon[23696]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:56 smithi081 ceph-mon[28146]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:57 smithi105 ceph-mon[24602]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:57 smithi081 ceph-mon[23696]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:57 smithi081 ceph-mon[28146]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:42:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:42:59 smithi105 ceph-mon[24602]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:42:59 smithi081 ceph-mon[23696]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:42:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:42:59 smithi081 ceph-mon[28146]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:01 smithi105 ceph-mon[24602]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:01 smithi081 ceph-mon[23696]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:01 smithi081 ceph-mon[28146]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:03 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:03 smithi105 ceph-mon[24602]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:03 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:03 smithi081 ceph-mon[23696]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:03.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:03 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:03.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:03 smithi081 ceph-mon[28146]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:03.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:43:03] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:43:06.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:06 smithi105 ceph-mon[24602]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:06 smithi081 ceph-mon[23696]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:06 smithi081 ceph-mon[28146]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:07 smithi105 ceph-mon[24602]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:07.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:07 smithi081 ceph-mon[23696]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:07.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:07 smithi081 ceph-mon[28146]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:08 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:08 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:08.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:08 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:09.625 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:09 smithi081 ceph-mon[23696]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:09.626 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:09 smithi081 ceph-mon[28146]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:09 smithi105 ceph-mon[24602]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:10.356 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:43:11.609 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:11 smithi081 ceph-mon[23696]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:11.610 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:11 smithi081 ceph-mon[28146]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:11 smithi105 ceph-mon[24602]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:12.354 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:43:12.354 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (9m) 2m ago 17m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:43:12.354 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (8m) 2m ago 16m 51.4M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:43:12.354 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (2m) 2m ago 16m 68.4M - 3.5 e1d6a67b021e cbe55a03e9d5 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (10m) 2m ago 21m 437M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (5m) 2m ago 23m 492M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (23m) 2m ago 23m 65.1M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (21m) 2m ago 21m 53.3M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (21m) 2m ago 21m 48.7M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (9m) 2m ago 17m 15.2M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (8m) 2m ago 17m 15.2M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (21m) 2m ago 21m 53.2M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (20m) 2m ago 20m 54.3M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (20m) 2m ago 20m 49.8M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (20m) 2m ago 20m 53.4M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (19m) 2m ago 19m 52.9M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:43:12.355 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (19m) 2m ago 19m 50.8M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:43:12.356 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (18m) 2m ago 18m 51.1M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:43:12.356 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (18m) 2m ago 18m 52.6M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:43:12.356 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (2m) 2m ago 17m 43.9M - 2.43.0 a07b618ecd1d 4c65e58981bf 2024-09-17T01:43:12.356 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (16m) 2m ago 16m 75.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:43:12.356 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (16m) 2m ago 16m 77.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:43:12.941 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:43:13.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[23696]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:43:13.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:13.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[23696]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:13.503 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:43:13] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-09-17T01:43:13.504 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[28146]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:43:13.504 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:13.504 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:13 smithi081 ceph-mon[28146]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:13 smithi105 ceph-mon[24602]: from='client.25138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:43:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:13 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:13 smithi105 ceph-mon[24602]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:14.922 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:43:14.922 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:43:14.923 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:43:14.924 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:43:14.924 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:43:14.924 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:43:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:14 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/163669627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:15.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:14 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/163669627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:14 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/163669627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:15.521 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph -s' 2024-09-17T01:43:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:16 smithi081 ceph-mon[23696]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:16.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:16 smithi081 ceph-mon[28146]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:16 smithi105 ceph-mon[24602]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: cluster: 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: id: ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: health: HEALTH_OK 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: services: 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-09-17T01:43:17.624 INFO:teuthology.orchestra.run.smithi081.stdout: mgr: y(active, since 3m), standbys: x 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: data: 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: pools: 6 pools, 161 pgs 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: objects: 241 objects, 585 KiB 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: pgs: 161 active+clean 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: io: 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-17T01:43:17.625 INFO:teuthology.orchestra.run.smithi081.stdout: 2024-09-17T01:43:17.626 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:17 smithi081 ceph-mon[23696]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:17.626 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:17 smithi081 ceph-mon[28146]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:17 smithi105 ceph-mon[24602]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:18.261 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:43:18.558 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:18 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/775831641' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:43:18.558 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:18 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/775831641' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:43:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:18 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/775831641' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T01:43:19.486 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:19 smithi081 ceph-mon[23696]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:19.486 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:19 smithi081 ceph-mon[28146]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:19.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:19 smithi105 ceph-mon[24602]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:20.234 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_OK 2024-09-17T01:43:20.585 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:20 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2963219357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:43:20.585 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:20 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2963219357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:43:20.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:20 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2963219357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:43:20.908 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-17T01:43:21.546 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:21 smithi081 ceph-mon[23696]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:21.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:21 smithi081 ceph-mon[28146]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:21 smithi105 ceph-mon[24602]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:22.841 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:22 smithi081 ceph-mon[23696]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:22.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:22 smithi081 ceph-mon[28146]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:22.912 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:43:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:22 smithi105 ceph-mon[24602]: from='mgr.25051 172.21.15.81:0/1014854388' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:23.524 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:43:23] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.43.0" 2024-09-17T01:43:23.546 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph mgr fail' 2024-09-17T01:43:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4193633892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[23696]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:23.834 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:23.834 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4193633892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:23.834 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:23 smithi081 ceph-mon[28146]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:23 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:23 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/4193633892' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:43:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:23 smithi105 ceph-mon[24602]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:26.611 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[23696]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:26.611 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:43:26.611 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[23696]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[28146]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mon[28146]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.256+0000 7fa2e532a640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ignoring --setuser ceph since I am not root 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ignoring --setgroup ceph since I am not root 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-mgr[60921]: -- 172.21.15.81:0/4125828174 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5614060510e0 con 0x56140602e800 2024-09-17T01:43:26.612 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.409+0000 7f7fb10e2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:43:26.613 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.465+0000 7f7fb10e2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:43:26.613 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.613+0000 7f7fb10e2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:43:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-mon[24602]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:43:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T01:43:26.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-mon[24602]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T01:43:26.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE Bus STOPPING 2024-09-17T01:43:26.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:43:26.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE Bus STOPPED 2024-09-17T01:43:26.903 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.682+0000 7f7fb10e2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:43:26.903 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.833+0000 7f7fb10e2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:43:26.934 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE Bus STARTING 2024-09-17T01:43:26.934 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE Serving on http://:::9283 2024-09-17T01:43:26.934 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:43:26] ENGINE Bus STARTED 2024-09-17T01:43:26.974 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 180' 2024-09-17T01:43:27.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:26.985+0000 7f7fb10e2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:43:27.509 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:27.333+0000 7f7fb10e2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:43:27.509 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:27.396+0000 7f7fb10e2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:43:27.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: mgrmap e33: x(active, starting, since 0.706092s) 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: Manager daemon x is now available 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.511 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:43:27.512 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: mgrmap e33: x(active, starting, since 0.706092s) 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: Manager daemon x is now available 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3829892792' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T01:43:27.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: mgrmap e33: x(active, starting, since 0.706092s) 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: Manager daemon x is now available 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:27 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T01:43:27.905 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:27.602+0000 7f7fb10e2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:43:28.258 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:43:28.053+0000 7f021d584640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:01:43:28] ENGINE Error in HTTPServer.serve 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._connections.run(self.expiration_interval) 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._run(expiration_interval) 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: s = self.context.wrap_socket( 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:43:28.259 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.sslsocket_class._create( 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.do_handshake() 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self._sslobj.do_handshake() 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:43:28.260 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: mgrmap e34: x(active, since 1.7035s) 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: mgrmap e34: x(active, since 1.7035s) 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: mgrmap e34: x(active, since 1.7035s) 2024-09-17T01:43:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:28 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:28.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:28.522+0000 7f7fb10e2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:43:28.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:28 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:28.592+0000 7f7fb10e2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: [17/Sep/2024:01:43:27] ENGINE Bus STARTING 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: [17/Sep/2024:01:43:28] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: [17/Sep/2024:01:43:28] ENGINE Error in HTTPServer.serve 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:43:29.578 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: [17/Sep/2024:01:43:28] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:43:29.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: [17/Sep/2024:01:43:28] ENGINE Bus STARTED 2024-09-17T01:43:29.580 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:29 smithi105 ceph-mon[24602]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:29.589 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: [17/Sep/2024:01:43:27] ENGINE Bus STARTING 2024-09-17T01:43:29.589 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: [17/Sep/2024:01:43:28] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:43:29.589 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: [17/Sep/2024:01:43:28] ENGINE Error in HTTPServer.serve 2024-09-17T01:43:29.589 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:43:29.590 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.do_handshake() 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: [17/Sep/2024:01:43:28] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: [17/Sep/2024:01:43:28] ENGINE Bus STARTED 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[23696]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: [17/Sep/2024:01:43:27] ENGINE Bus STARTING 2024-09-17T01:43:29.591 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: [17/Sep/2024:01:43:28] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: [17/Sep/2024:01:43:28] ENGINE Error in HTTPServer.serve 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:43:29.592 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.do_handshake() 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: [17/Sep/2024:01:43:28] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: [17/Sep/2024:01:43:28] ENGINE Bus STARTED 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-mon[28146]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.336+0000 7f7fb10e2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:43:29.593 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.465+0000 7f7fb10e2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:43:29.900 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.610+0000 7f7fb10e2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:43:29.900 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.675+0000 7f7fb10e2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:43:29.900 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.832+0000 7f7fb10e2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:43:29.900 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.902+0000 7f7fb10e2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:43:30.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:29 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:29.963+0000 7f7fb10e2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:43:30.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.139+0000 7f7fb10e2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-mon[23696]: mgrmap e35: x(active, since 3s) 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-mon[28146]: mgrmap e35: x(active, since 3s) 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.212+0000 7f7fb10e2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.296+0000 7f7fb10e2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.389+0000 7f7fb10e2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.449+0000 7f7fb10e2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:43:30.510 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:30.511+0000 7f7fb10e2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:43:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:30 smithi105 ceph-mon[24602]: mgrmap e35: x(active, since 3s) 2024-09-17T01:43:31.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-mon[23696]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:31.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-mon[28146]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:31.532 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:31.341+0000 7f7fb10e2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:43:31.532 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:43:31] ENGINE Bus STARTING 2024-09-17T01:43:31.532 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: CherryPy Checker: 2024-09-17T01:43:31.533 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: The Application mounted at '' has an empty config. 2024-09-17T01:43:31.533 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:43:31.533 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:43:31] ENGINE Serving on http://:::9283 2024-09-17T01:43:31.533 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:43:31] ENGINE Bus STARTED 2024-09-17T01:43:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:31 smithi105 ceph-mon[24602]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: mgrmap e36: x(active, since 5s) 2024-09-17T01:43:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: Standby manager daemon y started 2024-09-17T01:43:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: mgrmap e36: x(active, since 5s) 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: Standby manager daemon y started 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:43:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:32 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: mgrmap e36: x(active, since 5s) 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: Standby manager daemon y started 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T01:43:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:43:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:32.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:32 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:33 smithi105 ceph-mon[24602]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 24 op/s 2024-09-17T01:43:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:33 smithi105 ceph-mon[24602]: mgrmap e37: x(active, since 6s), standbys: y 2024-09-17T01:43:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:33 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:43:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:33 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:33 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:43:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-17T01:43:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[23696]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 24 op/s 2024-09-17T01:43:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[23696]: mgrmap e37: x(active, since 6s), standbys: y 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[28146]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 24 op/s 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[28146]: mgrmap e37: x(active, since 6s), standbys: y 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:33 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:34 smithi105 ceph-mon[24602]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:34 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:34 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:34 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:34.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:34 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[23696]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:34.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[28146]: from='client.25123 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:43:34.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:34.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:34.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:34 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:35 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:35.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:43:35.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:35.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:35 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:36 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:36 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:36 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:43:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:36 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:36 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:37.606 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:37 smithi081 ceph-mon[23696]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:43:37.607 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:37 smithi081 ceph-mon[28146]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:43:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-mon[24602]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:43:37.697+0000 7f0251028640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:43:38.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:38.137 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.wait_async( 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return future.result(timeout) 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:38.138 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.__get_result() 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise self._exception 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: out, err, code = await self._run_cephadm( 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise OrchestratorError( 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:38.139 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:38.140 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:38.141 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:38.141 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:38.141 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:38.141 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:38.141 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.584 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:39 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.585 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.586 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:39.587 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.588 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.589 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:39 smithi105 ceph-mon[24602]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:39 smithi105 ceph-mon[24602]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:43:39.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:39 smithi105 ceph-mon[24602]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:43:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:43:39.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.684 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:39.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:43:39.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.688 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.689 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.690 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[23696]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:43:39.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[23696]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:43:39.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[23696]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:43:39.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:43:39.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:39.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:43:39.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:39.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:39.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:39.693 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[28146]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:43:39.694 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[28146]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:43:39.694 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:39 smithi081 ceph-mon[28146]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:43:41.247 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:43:41.631 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-mon[24602]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:41.632 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.341Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:43:41.632 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-17T01:43:41.632 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-17T01:43:41.632 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-17T01:43:41.633 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T01:43:41.634 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-17T01:43:41.634 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[53158]: ts=2024-09-17T01:43:41.342Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-17T01:43:41.634 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 podman[56104]: 2024-09-17 01:43:41.351995177 +0000 UTC m=+0.110155477 container died 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:41 smithi081 ceph-mon[23696]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:41 smithi081 ceph-mon[28146]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:41.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 podman[56104]: 2024-09-17 01:43:41.741860655 +0000 UTC m=+0.500020961 container cleanup 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:41.885 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 bash[56104]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T01:43:42.248 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:41 smithi105 podman[56116]: 2024-09-17 01:43:41.993520413 +0000 UTC m=+0.643829001 container remove 4c65e58981bf07588107b8f5bea1ace09594284f36373c590c18d49fe4d31aec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:42.248 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T01:43:42.248 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 systemd[1]: Stopped Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:43:42.248 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.734s CPU time. 2024-09-17T01:43:42.249 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 systemd[1]: Starting Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:43:42.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:42.506 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 podman[56209]: 2024-09-17 01:43:42.272904917 +0000 UTC m=+0.017274179 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-17T01:43:42.506 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 podman[56209]: 2024-09-17 01:43:42.403123069 +0000 UTC m=+0.147492358 container create 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:42 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:42 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 podman[56209]: 2024-09-17 01:43:42.613514194 +0000 UTC m=+0.357883448 container init 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 podman[56209]: 2024-09-17 01:43:42.616415946 +0000 UTC m=+0.360785197 container start 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.639Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.639Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.640Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.640Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.640Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.641Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.641Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.642Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-17T01:43:42.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.643Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.644Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.644Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.426µs 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.644Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.651Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.665Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.675Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.677Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.677Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.677Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.693µs wal_replay_duration=33.205714ms wbl_replay_duration=86ns total_replay_duration=33.247153ms 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.680Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.680Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.680Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T01:43:42.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 bash[56209]: 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f 2024-09-17T01:43:42.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:43:42.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.704Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.266595ms db_storage=756ns remote_storage=1.063µs web_handler=400ns query_engine=778ns scrape=435.79µs scrape_sd=94.452µs notify=26.537µs notify_sd=19.347µs rules=22.361319ms tracing=6.633µs 2024-09-17T01:43:42.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.704Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-17T01:43:42.888 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:43:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:43:42.704Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-17T01:43:43.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:43 smithi105 ceph-mon[24602]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:43.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:43 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:43 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:43 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[23696]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[28146]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:43:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:43.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:43 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:44 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:44 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:44 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:43:45.616 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:45 smithi105 ceph-mon[24602]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:45 smithi081 ceph-mon[23696]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:45 smithi081 ceph-mon[28146]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:47 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:43:47.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:47 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:43:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:48 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:48 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:48 smithi105 ceph-mon[24602]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:48.637 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:48.638 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:48.638 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[28146]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:48.638 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:43:48.638 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:43:48.638 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:48 smithi081 ceph-mon[23696]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:43:51.425 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:51 smithi081 ceph-mon[23696]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:51.425 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:51 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3550155967' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:43:51.425 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:51 smithi081 ceph-mon[28146]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:51.425 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:51 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3550155967' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:43:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-mon[24602]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:43:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3550155967' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:43:51.177+0000 7f0251028640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:51.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:51.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:51.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.wait_async( 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return future.result(timeout) 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:51.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.__get_result() 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise self._exception 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: out, err, code = await self._run_cephadm( 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise OrchestratorError( 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:51.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:51.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:51.643 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:52 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:43:52.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:52 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2913710194' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:52.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:52 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:52.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:52.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:52.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:43:52.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:43:52.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:43:52.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:43:52.692 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi-foo-smithi081-umxnyg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: /usr/bin/podman: stderr Error: no such container ceph-ded6f264-7492-11ef-bceb-c7b262605968-iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:43:52.693 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.694 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2913710194' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:43:52.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2913710194' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:52.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:52 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]: dispatch 2024-09-17T01:43:53.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:43:53] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-09-17T01:43:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]': finished 2024-09-17T01:43:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-mon[24602]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T01:43:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-mon[24602]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-17T01:43:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3450426365' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:53 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]': finished 2024-09-17T01:43:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[23696]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T01:43:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[23696]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3450426365' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3498913491"}]': finished 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[28146]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[28146]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3450426365' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:53 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]: dispatch 2024-09-17T01:43:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:54 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]': finished 2024-09-17T01:43:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:54 smithi105 ceph-mon[24602]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T01:43:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:54 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/282229143' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:54 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]': finished 2024-09-17T01:43:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[23696]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/282229143' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3346517331"}]': finished 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[28146]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/282229143' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:54 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]: dispatch 2024-09-17T01:43:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:55 smithi105 ceph-mon[24602]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T01:43:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:55 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]': finished 2024-09-17T01:43:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:55 smithi105 ceph-mon[24602]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T01:43:55.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:55 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/487574572' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:55.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:55 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[23696]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]': finished 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[23696]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/487574572' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[28146]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/1452400910"}]': finished 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[28146]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/487574572' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:55 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]: dispatch 2024-09-17T01:43:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:56 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]': finished 2024-09-17T01:43:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:56 smithi105 ceph-mon[24602]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T01:43:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:56 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]: dispatch 2024-09-17T01:43:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]': finished 2024-09-17T01:43:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[23696]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T01:43:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]: dispatch 2024-09-17T01:43:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1520381824"}]': finished 2024-09-17T01:43:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[28146]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T01:43:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:56 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]: dispatch 2024-09-17T01:43:57.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:43:57.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]': finished 2024-09-17T01:43:57.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T01:43:57.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1199685833' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:57 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:43:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]': finished 2024-09-17T01:43:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T01:43:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1199685833' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2101364809' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/1452400910"}]': finished 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1199685833' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]: dispatch 2024-09-17T01:43:57.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:57 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:43:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:58 smithi105 ceph-mon[24602]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]': finished 2024-09-17T01:43:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:58 smithi105 ceph-mon[24602]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T01:43:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:58 smithi081 ceph-mon[23696]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]': finished 2024-09-17T01:43:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:58 smithi081 ceph-mon[23696]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T01:43:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:58 smithi081 ceph-mon[28146]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/249562800"}]': finished 2024-09-17T01:43:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:58 smithi081 ceph-mon[28146]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T01:43:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:43:59 smithi105 ceph-mon[24602]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:43:59 smithi081 ceph-mon[23696]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:43:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:43:59 smithi081 ceph-mon[28146]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:01 smithi105 ceph-mon[24602]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-09-17T01:44:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:01 smithi081 ceph-mon[23696]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-09-17T01:44:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:01 smithi081 ceph-mon[28146]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-09-17T01:44:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:02 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:03 smithi105 ceph-mon[24602]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:44:03.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:03] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:44:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:03 smithi081 ceph-mon[23696]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:44:03.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:03 smithi081 ceph-mon[28146]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:44:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:04 smithi105 ceph-mon[24602]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:04 smithi081 ceph-mon[23696]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:04 smithi081 ceph-mon[28146]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:06 smithi105 ceph-mon[24602]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:44:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:06 smithi081 ceph-mon[23696]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:44:06.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:06 smithi081 ceph-mon[28146]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:44:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:08 smithi105 ceph-mon[24602]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-17T01:44:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:08 smithi081 ceph-mon[23696]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-17T01:44:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:08 smithi081 ceph-mon[28146]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-09-17T01:44:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:11 smithi105 ceph-mon[24602]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:11 smithi081 ceph-mon[23696]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:11 smithi081 ceph-mon[28146]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:12 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:12 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:12 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:12 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:12 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:12 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:13 smithi105 ceph-mon[24602]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:13.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:13] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:44:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:13 smithi081 ceph-mon[23696]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:13.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:13 smithi081 ceph-mon[28146]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:14 smithi105 ceph-mon[24602]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:14 smithi081 ceph-mon[23696]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:14 smithi081 ceph-mon[28146]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:16 smithi105 ceph-mon[24602]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:16 smithi081 ceph-mon[23696]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:16 smithi081 ceph-mon[28146]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:18 smithi105 ceph-mon[24602]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:18 smithi081 ceph-mon[23696]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:18.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:18 smithi081 ceph-mon[28146]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:21.621 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:21 smithi081 ceph-mon[23696]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:21.621 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:21 smithi081 ceph-mon[28146]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:21 smithi105 ceph-mon[24602]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:22 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:22 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:22 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:23 smithi105 ceph-mon[24602]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:23.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:23] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:44:23.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:23 smithi081 ceph-mon[23696]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:23.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:23 smithi081 ceph-mon[28146]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:24 smithi105 ceph-mon[24602]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:24 smithi081 ceph-mon[28146]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:24 smithi081 ceph-mon[23696]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:26 smithi105 ceph-mon[24602]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:26 smithi081 ceph-mon[23696]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:26 smithi081 ceph-mon[28146]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:28 smithi105 ceph-mon[24602]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:28 smithi081 ceph-mon[23696]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:28.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:28 smithi081 ceph-mon[28146]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:31 smithi105 ceph-mon[24602]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:31 smithi081 ceph-mon[23696]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:31 smithi081 ceph-mon[28146]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:32 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:32 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:32 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:33 smithi105 ceph-mon[24602]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:33.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:33] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.43.0" 2024-09-17T01:44:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:33 smithi081 ceph-mon[23696]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:33.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:33 smithi081 ceph-mon[28146]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:34 smithi105 ceph-mon[24602]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:34 smithi081 ceph-mon[23696]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:34 smithi081 ceph-mon[28146]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:36 smithi105 ceph-mon[24602]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:36 smithi081 ceph-mon[23696]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:36 smithi081 ceph-mon[28146]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:38 smithi105 ceph-mon[24602]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:38 smithi081 ceph-mon[23696]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:38.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:38 smithi081 ceph-mon[28146]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:41 smithi105 ceph-mon[24602]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:41 smithi081 ceph-mon[28146]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:41 smithi081 ceph-mon[23696]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:42 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:42 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:42 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:42 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:42.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:42 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:42.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:42 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:43 smithi105 ceph-mon[24602]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:43.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:43] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.43.0" 2024-09-17T01:44:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:43 smithi081 ceph-mon[23696]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:43.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:43 smithi081 ceph-mon[28146]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:44 smithi105 ceph-mon[24602]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:44 smithi081 ceph-mon[23696]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:44 smithi081 ceph-mon[28146]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:46 smithi105 ceph-mon[24602]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:46 smithi081 ceph-mon[23696]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:46.703 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:46 smithi081 ceph-mon[28146]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:48.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:48 smithi105 ceph-mon[24602]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:48.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:48 smithi081 ceph-mon[23696]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:48.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:48 smithi081 ceph-mon[28146]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:51 smithi105 ceph-mon[24602]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:51 smithi081 ceph-mon[23696]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:51.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:51 smithi081 ceph-mon[28146]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:52.392 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:52.392 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:44:52.393 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:44:52.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:52 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:44:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:44:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:44:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:44:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:52 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:44:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:44:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:44:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-mon[24602]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:44:53.136+0000 7f0251028640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:53.386 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:53.387 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.wait_async( 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return future.result(timeout) 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.__get_result() 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise self._exception 2024-09-17T01:44:53.388 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: out, err, code = await self._run_cephadm( 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise OrchestratorError( 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:53.389 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:53.390 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:53.391 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:44:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:44:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[23696]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:53.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:44:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:44:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:53 smithi081 ceph-mon[28146]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:53.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:44:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:44:53] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.43.0" 2024-09-17T01:44:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:54 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:44:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:44:54.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:54 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:44:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:54 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:44:54.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:44:54.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:44:54.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:44:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:55 smithi105 ceph-mon[24602]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:55 smithi081 ceph-mon[28146]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:55 smithi081 ceph-mon[23696]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:56 smithi105 ceph-mon[24602]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:56 smithi081 ceph-mon[23696]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:56 smithi081 ceph-mon[28146]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:44:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:57 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:57 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:57 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:44:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:44:58 smithi105 ceph-mon[24602]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:44:58 smithi081 ceph-mon[23696]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:44:58.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:44:58 smithi081 ceph-mon[28146]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:01 smithi105 ceph-mon[24602]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:01 smithi081 ceph-mon[23696]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:01 smithi081 ceph-mon[28146]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:02 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:02.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:03 smithi105 ceph-mon[24602]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:03.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:03] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.43.0" 2024-09-17T01:45:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:03 smithi081 ceph-mon[23696]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:03.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:03 smithi081 ceph-mon[28146]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:04 smithi105 ceph-mon[24602]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:04 smithi081 ceph-mon[23696]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:04.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:04 smithi081 ceph-mon[28146]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:06 smithi105 ceph-mon[24602]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:06 smithi081 ceph-mon[23696]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:06 smithi081 ceph-mon[28146]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:08 smithi105 ceph-mon[24602]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:08 smithi081 ceph-mon[23696]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:08 smithi081 ceph-mon[28146]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:11 smithi105 ceph-mon[24602]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:11 smithi081 ceph-mon[23696]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:11.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:11 smithi081 ceph-mon[28146]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:12 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:12.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:12 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:12 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:12 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:12 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:12 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:13 smithi105 ceph-mon[24602]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:13.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:13] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:45:13.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:13 smithi081 ceph-mon[28146]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:13 smithi081 ceph-mon[23696]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:14.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:14 smithi105 ceph-mon[24602]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:14 smithi081 ceph-mon[23696]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:14 smithi081 ceph-mon[28146]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:16 smithi105 ceph-mon[24602]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:16 smithi081 ceph-mon[23696]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:16 smithi081 ceph-mon[28146]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:18 smithi105 ceph-mon[24602]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:18 smithi081 ceph-mon[23696]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:18.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:18 smithi081 ceph-mon[28146]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:21.633 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:21 smithi081 ceph-mon[23696]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:21.633 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:21 smithi081 ceph-mon[28146]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:21 smithi105 ceph-mon[24602]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:22 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:22 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:22 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:23 smithi105 ceph-mon[24602]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:23.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:23] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-09-17T01:45:23.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:23 smithi081 ceph-mon[23696]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:23.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:23 smithi081 ceph-mon[28146]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:24.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:24 smithi105 ceph-mon[24602]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:24 smithi081 ceph-mon[23696]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:24.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:24 smithi081 ceph-mon[28146]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:26.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:26 smithi105 ceph-mon[24602]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:26 smithi081 ceph-mon[23696]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:26 smithi081 ceph-mon[28146]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:27.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:27.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:28 smithi081 ceph-mon[23696]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:28 smithi081 ceph-mon[28146]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:28.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:28 smithi105 ceph-mon[24602]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:31 smithi105 ceph-mon[24602]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:31 smithi081 ceph-mon[23696]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:31.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:31 smithi081 ceph-mon[28146]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:32 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:32 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:32 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:33 smithi105 ceph-mon[24602]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:33.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:33] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.43.0" 2024-09-17T01:45:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:33 smithi081 ceph-mon[23696]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:33 smithi081 ceph-mon[28146]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:34.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:34 smithi105 ceph-mon[24602]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:34.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:34 smithi081 ceph-mon[23696]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:34.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:34 smithi081 ceph-mon[28146]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:36.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:36 smithi105 ceph-mon[24602]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:36 smithi081 ceph-mon[23696]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:36.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:36 smithi081 ceph-mon[28146]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:38 smithi105 ceph-mon[24602]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:38.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:38 smithi081 ceph-mon[28146]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:38 smithi081 ceph-mon[23696]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:41 smithi105 ceph-mon[24602]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:41 smithi081 ceph-mon[23696]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:41.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:41 smithi081 ceph-mon[28146]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:42 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:42 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:42 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:42 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:42 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:42 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:43 smithi105 ceph-mon[24602]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:43.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:43] "GET /metrics HTTP/1.1" 200 37923 "" "Prometheus/2.43.0" 2024-09-17T01:45:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:43 smithi081 ceph-mon[23696]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:43.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:43 smithi081 ceph-mon[28146]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:44.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:44 smithi081 ceph-mon[23696]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:44.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:44 smithi081 ceph-mon[28146]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:44.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:44 smithi105 ceph-mon[24602]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:46 smithi105 ceph-mon[24602]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:46 smithi081 ceph-mon[23696]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:46 smithi081 ceph-mon[28146]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:48.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:48 smithi081 ceph-mon[23696]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:48.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:48 smithi081 ceph-mon[28146]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:48 smithi105 ceph-mon[24602]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:51 smithi105 ceph-mon[24602]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:51.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:51 smithi081 ceph-mon[23696]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:51.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:51 smithi081 ceph-mon[28146]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:52 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:52 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:52 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:45:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:53 smithi105 ceph-mon[24602]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:53.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:45:53] "GET /metrics HTTP/1.1" 200 37923 "" "Prometheus/2.43.0" 2024-09-17T01:45:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:53 smithi081 ceph-mon[23696]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:53.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:53 smithi081 ceph-mon[28146]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:54.347 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:54.347 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:45:54.347 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:45:54.347 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.348 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:54 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:45:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:45:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:45:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:54 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:45:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:45:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:45:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[23696]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:55.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:45:55.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:45:55.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:55 smithi081 ceph-mon[28146]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:45:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:45:55.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-mon[24602]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:55.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:45:55.192+0000 7f0251028640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:55.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:55.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:55.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.wait_async( 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:45:55.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return future.result(timeout) 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.__get_result() 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise self._exception 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: out, err, code = await self._run_cephadm( 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise OrchestratorError( 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:55.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:55.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:55.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:55.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:55.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:55.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:55.642 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:45:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:56 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:45:56.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:56 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.681 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:45:56.682 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.683 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.684 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:56 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.685 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.686 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:45:56.687 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:45:56.688 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:56.689 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:45:56.690 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:45:56.691 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:45:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:57 smithi105 ceph-mon[24602]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:57 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:57 smithi081 ceph-mon[23696]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:57 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:57 smithi081 ceph-mon[28146]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:45:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:57 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:45:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:45:59 smithi105 ceph-mon[24602]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:45:59 smithi081 ceph-mon[23696]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:45:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:45:59 smithi081 ceph-mon[28146]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:01 smithi105 ceph-mon[24602]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:01 smithi081 ceph-mon[23696]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:01.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:01 smithi081 ceph-mon[28146]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:02 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:03 smithi105 ceph-mon[24602]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:03.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:03] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:46:03.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:03 smithi081 ceph-mon[23696]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:03.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:03 smithi081 ceph-mon[28146]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:04.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:04 smithi081 ceph-mon[23696]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:04.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:04 smithi081 ceph-mon[28146]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:04.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:04 smithi105 ceph-mon[24602]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:06 smithi081 ceph-mon[23696]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:06 smithi081 ceph-mon[28146]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:06.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:06 smithi105 ceph-mon[24602]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:08.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:08 smithi081 ceph-mon[23696]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:08.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:08 smithi081 ceph-mon[28146]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:08.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:08 smithi105 ceph-mon[24602]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:11 smithi105 ceph-mon[24602]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:11 smithi081 ceph-mon[23696]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:11.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:11 smithi081 ceph-mon[28146]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:12 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:12 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:12 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:12 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:12 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:12 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:13 smithi105 ceph-mon[24602]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:13.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:13] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:46:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:13 smithi081 ceph-mon[23696]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:13.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:13 smithi081 ceph-mon[28146]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:14.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:14 smithi081 ceph-mon[23696]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:14.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:14 smithi081 ceph-mon[28146]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:14.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:14 smithi105 ceph-mon[24602]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:16 smithi081 ceph-mon[23696]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:16.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:16 smithi081 ceph-mon[28146]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:16.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:16 smithi105 ceph-mon[24602]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:18 smithi081 ceph-mon[23696]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:18.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:18 smithi081 ceph-mon[28146]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:18.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:18 smithi105 ceph-mon[24602]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:21.634 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:21 smithi081 ceph-mon[23696]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:21.634 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:21 smithi081 ceph-mon[28146]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:21 smithi105 ceph-mon[24602]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:22 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:22.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:22 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:22.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:22 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:23.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:23 smithi105 ceph-mon[24602]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:23.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:23] "GET /metrics HTTP/1.1" 200 37921 "" "Prometheus/2.43.0" 2024-09-17T01:46:23.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:23 smithi081 ceph-mon[28146]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:23.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:23 smithi081 ceph-mon[23696]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:24 smithi081 ceph-mon[23696]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:24.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:24 smithi081 ceph-mon[28146]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:24.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:24 smithi105 ceph-mon[24602]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:26 smithi081 ceph-mon[23696]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:26 smithi081 ceph-mon[28146]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:26.687 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:26 smithi105 ceph-mon[24602]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:27.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:27 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:27.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:27 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:27.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:27 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:28.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:28 smithi081 ceph-mon[23696]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:28 smithi081 ceph-mon[28146]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:28 smithi105 ceph-mon[24602]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:29.165 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (12m) 2m ago 20m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (12m) 2m ago 20m 51.9M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (5m) 2m ago 19m 47.1M - 3.5 e1d6a67b021e cbe55a03e9d5 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (13m) 2m ago 24m 490M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (8m) 2m ago 26m 142M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (26m) 2m ago 26m 70.0M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:46:31.201 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (25m) 2m ago 25m 55.3M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (25m) 2m ago 25m 50.7M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (12m) 2m ago 21m 15.6M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (12m) 2m ago 21m 15.6M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (24m) 2m ago 24m 53.2M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (24m) 2m ago 24m 54.5M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (23m) 2m ago 23m 49.9M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (23m) 2m ago 23m 53.5M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (23m) 2m ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (22m) 2m ago 22m 51.1M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (22m) 2m ago 22m 51.4M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (21m) 2m ago 21m 52.9M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (2m) 2m ago 21m 40.6M - 2.43.0 a07b618ecd1d 65d14fae0658 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (19m) 2m ago 19m 76.5M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:46:31.202 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (19m) 2m ago 19m 77.9M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:46:31.500 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:31 smithi081 ceph-mon[23696]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:31.500 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:31 smithi081 ceph-mon[28146]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:31 smithi105 ceph-mon[24602]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:31.832 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ls' 2024-09-17T01:46:32.388 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:32 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:32.388 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:32 smithi081 ceph-mon[23696]: from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:32.388 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:32 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:32.388 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:32 smithi081 ceph-mon[28146]: from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:32 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:32 smithi105 ceph-mon[24602]: from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:33.410 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:33 smithi081 ceph-mon[23696]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:33.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:33 smithi081 ceph-mon[28146]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:33 smithi105 ceph-mon[24602]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:33.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:33] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi081=a;count:1 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:grafana ?:3000 1/1 2m ago 21m smithi105=a;count:1 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo ?:5000 1/1 3m ago 19m count:1 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:mgr 2/2 3m ago 24m smithi081=y;smithi105=x;count:2 2024-09-17T01:46:33.854 INFO:teuthology.orchestra.run.smithi081.stdout:mon 3/3 3m ago 25m smithi081:172.21.15.81=a;smithi081:[v2:172.21.15.81:3301,v1:172.21.15.81:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T01:46:33.855 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi081=a;smithi105=b;count:2 2024-09-17T01:46:33.855 INFO:teuthology.orchestra.run.smithi081.stdout:osd 8 3m ago - 2024-09-17T01:46:33.855 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus ?:9095 1/1 2m ago 21m smithi105=a;count:1 2024-09-17T01:46:33.855 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-09-17T01:46:34.514 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:34 smithi081 ceph-mon[23696]: from='client.15360 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:34.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:34 smithi081 ceph-mon[23696]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:34.515 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:34 smithi081 ceph-mon[28146]: from='client.15360 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:34.515 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:34 smithi081 ceph-mon[28146]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:34.538 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:46:34.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:34 smithi105 ceph-mon[24602]: from='client.15360 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:34.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:34 smithi105 ceph-mon[24602]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:46:36.503 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:46:36.504 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:46:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:36 smithi081 ceph-mon[23696]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:36 smithi081 ceph-mon[28146]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:36.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:36 smithi105 ceph-mon[24602]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:37.110 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-17T01:46:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:37 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2725746505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:37.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:37 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2725746505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:37.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:37 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2725746505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:38.547 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:38 smithi081 ceph-mon[23696]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:38.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:38 smithi081 ceph-mon[28146]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:38.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:38 smithi105 ceph-mon[24602]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:39.624 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:39 smithi081 ceph-mon[23696]: from='client.15372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:39.624 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:39 smithi081 ceph-mon[28146]: from='client.15372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:39.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:39 smithi105 ceph-mon[24602]: from='client.15372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:40.387 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:40.989 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.561 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.562 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:41 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:46:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:41 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[23696]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[23696]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[23696]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[28146]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[28146]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[28146]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:46:42.468 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:42.469 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:42 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:42 smithi105 ceph-mon[24602]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:42 smithi105 ceph-mon[24602]: Reconfiguring iscsi.foo.smithi081.umxnyg (dependencies changed)... 2024-09-17T01:46:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:42 smithi105 ceph-mon[24602]: Reconfiguring daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:46:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:42 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:42.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:42 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:43.492 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:43 smithi081 ceph-mon[23696]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:43.492 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:43 smithi081 ceph-mon[28146]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-mon[24602]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:43.312+0000 7f0251028640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:43.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:43.637 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.wait_async( 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:46:43.638 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return future.result(timeout) 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return self.__get_result() 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise self._exception 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: out, err, code = await self._run_cephadm( 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: raise OrchestratorError( 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Creating ceph-iscsi config... 2024-09-17T01:46:43.639 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: Traceback (most recent call last): 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:43.640 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: return _run_code(code, main_globals, None, 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: exec(code, run_globals) 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:43.641 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:43] "GET /metrics HTTP/1.1" 200 37917 "" "Prometheus/2.43.0" 2024-09-17T01:46:43.949 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:46:44.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:44 smithi081 ceph-mon[23696]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.478 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:46:44.479 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.wait_async( 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: return future.result(timeout) 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.__get_result() 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise self._exception 2024-09-17T01:46:44.480 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.481 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.482 INFO:journalctl@ceph.mon.a.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:44 smithi081 ceph-mon[28146]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.484 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.485 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.wait_async( 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: return future.result(timeout) 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.__get_result() 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:46:44.486 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise self._exception 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: raise OrchestratorError( 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.487 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.488 INFO:journalctl@ceph.mon.c.smithi081.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.615 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:46:44.616 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (12m) 3m ago 20m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:46:44.616 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (12m) 2m ago 20m 51.9M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:46:44.616 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 running (6m) 3m ago 19m 47.1M - 3.5 e1d6a67b021e cbe55a03e9d5 2024-09-17T01:46:44.616 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283 running (13m) 2m ago 25m 490M - 19.1.1-330-gc2ddf972 ebb0615149de d4a850b74b11 2024-09-17T01:46:44.616 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (9m) 3m ago 26m 142M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:46:44.617 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (26m) 3m ago 26m 70.0M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:46:44.617 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (25m) 2m ago 25m 55.3M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:46:44.617 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (25m) 3m ago 25m 50.7M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:46:44.617 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (12m) 3m ago 21m 15.6M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:46:44.617 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (12m) 2m ago 21m 15.6M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (24m) 3m ago 24m 53.2M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (24m) 3m ago 24m 54.5M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (24m) 3m ago 24m 49.9M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (23m) 3m ago 23m 53.5M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (23m) 2m ago 23m 53.1M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:46:44.618 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (22m) 2m ago 22m 51.1M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:46:44.619 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (22m) 2m ago 22m 51.4M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:46:44.619 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (22m) 2m ago 22m 52.9M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:46:44.619 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (3m) 2m ago 21m 40.6M - 2.43.0 a07b618ecd1d 65d14fae0658 2024-09-17T01:46:44.619 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (19m) 3m ago 19m 76.5M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:46:44.620 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (19m) 2m ago 19m 77.9M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:46:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:44 smithi105 ceph-mon[24602]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.636 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.637 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.638 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.639 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-09-17T01:46:44.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-17T01:46:44.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-09-17T01:46:44.640 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.wait_async( 2024-09-17T01:46:44.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-09-17T01:46:44.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.event_loop.get_result(coro, timeout) 2024-09-17T01:46:44.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-09-17T01:46:44.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: return future.result(timeout) 2024-09-17T01:46:44.641 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-09-17T01:46:44.642 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.__get_result() 2024-09-17T01:46:44.642 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-09-17T01:46:44.642 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise self._exception 2024-09-17T01:46:44.642 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-09-17T01:46:44.643 INFO:journalctl@ceph.mon.b.smithi105.stdout: out, err, code = await self._run_cephadm( 2024-09-17T01:46:44.643 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-09-17T01:46:44.643 INFO:journalctl@ceph.mon.b.smithi105.stdout: raise OrchestratorError( 2024-09-17T01:46:44.644 INFO:journalctl@ceph.mon.b.smithi105.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:46:44.644 INFO:journalctl@ceph.mon.b.smithi105.stdout: Creating ceph-iscsi config... 2024-09-17T01:46:44.644 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:46:44.645 INFO:journalctl@ceph.mon.b.smithi105.stdout: Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:46:44.645 INFO:journalctl@ceph.mon.b.smithi105.stdout: Non-zero exit code 1 from systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:46:44.645 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.645 INFO:journalctl@ceph.mon.b.smithi105.stdout: systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: exec(code, run_globals) 2024-09-17T01:46:44.646 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-09-17T01:46:44.647 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:46:44.648 INFO:journalctl@ceph.mon.b.smithi105.stdout: RuntimeError: Failed command: systemctl restart ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:46:44.648 INFO:journalctl@ceph.mon.b.smithi105.stdout: See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:46:45.016 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:46:45.016 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:46:45.016 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:46:45.017 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:46:45.018 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:46:45.018 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:46:45.018 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:46:45.018 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:46:45.018 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [ 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr" 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f image", 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:46:45.352 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:46:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='client.35009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='client.15384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:46:45.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/1437509657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='client.35009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='client.15384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/1437509657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='client.35009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='client.15384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' 2024-09-17T01:46:45.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:45.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='mgr.34865 172.21.15.105:0/1043207717' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T01:46:45.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:46:45.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/1437509657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:46:46.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:45.852+0000 7f02bbd5b640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T01:46:46.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ignoring --setuser ceph since I am not root 2024-09-17T01:46:46.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ignoring --setgroup ceph since I am not root 2024-09-17T01:46:46.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:45 smithi105 ceph-mgr[44255]: -- 172.21.15.105:0/3229272558 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5642c6f0f0e0 con 0x5642c6eed400 2024-09-17T01:46:46.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.002+0000 7f4d2a70a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:46:46.136 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.058+0000 7f4d2a70a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:46:46.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:45] ENGINE Bus STOPPING 2024-09-17T01:46:46.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:46:46.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:46] ENGINE Bus STOPPED 2024-09-17T01:46:46.453 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.203+0000 7f4d2a70a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.268+0000 7f4d2a70a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.417+0000 7f4d2a70a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Failing over to other MGR 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.34865 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: mgrmap e38: y(active, starting, since 0.995223s) 2024-09-17T01:46:46.454 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:46:46.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:46:46.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-mon[24602]: Manager daemon y is now available 2024-09-17T01:46:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:46:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:46:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Failing over to other MGR 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.34865 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: mgrmap e38: y(active, starting, since 0.995223s) 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:46:46.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[23696]: Manager daemon y is now available 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='client.25297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:46:46.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Failing over to other MGR 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.34865 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: mgrmap e38: y(active, starting, since 0.995223s) 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:46:46.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:46:46.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-mon[28146]: Manager daemon y is now available 2024-09-17T01:46:46.685 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:46] ENGINE Bus STARTING 2024-09-17T01:46:46.685 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:46] ENGINE Serving on http://:::9283 2024-09-17T01:46:46.685 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:46:46] ENGINE Bus STARTED 2024-09-17T01:46:46.884 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.561+0000 7f4d2a70a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:46:47.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.886+0000 7f4d2a70a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:46:47.135 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:46.954+0000 7f4d2a70a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:46:47.473 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:47.153+0000 7f4d2a70a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:47 smithi105 ceph-mon[24602]: mgrmap e39: y(active, since 1.99753s) 2024-09-17T01:46:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[23696]: mgrmap e39: y(active, since 1.99753s) 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:46:47.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-mon[28146]: mgrmap e39: y(active, since 1.99753s) 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:46:47.842+0000 7f7f0809f640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:01:46:47] ENGINE Error in HTTPServer.serve 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._connections.run(self.expiration_interval) 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._run(expiration_interval) 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:46:48.105 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s = self.context.wrap_socket( 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.sslsocket_class._create( 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self.do_handshake() 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._sslobj.do_handshake() 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:46:48.106 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:46:47 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:46:48.337 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:48 smithi105 ceph-mon[24602]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:48.337 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:48.066+0000 7f4d2a70a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:46:48.337 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:48.134+0000 7f4d2a70a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:46:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:48 smithi081 ceph-mon[23696]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:48.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:48 smithi081 ceph-mon[28146]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:49.188 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:48.911+0000 7f4d2a70a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:46:49.188 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.030+0000 7f4d2a70a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:46:49.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: [17/Sep/2024:01:46:47] ENGINE Bus STARTING 2024-09-17T01:46:49.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: [17/Sep/2024:01:46:47] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:46:49.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: [17/Sep/2024:01:46:47] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:46:49.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: [17/Sep/2024:01:46:47] ENGINE Bus STARTED 2024-09-17T01:46:49.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: [17/Sep/2024:01:46:47] ENGINE Error in HTTPServer.serve 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T01:46:49.503 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:46:49.504 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:46:49.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:49.506 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-mon[24602]: mgrmap e40: y(active, since 3s) 2024-09-17T01:46:49.506 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.190+0000 7f4d2a70a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:46:49.506 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.261+0000 7f4d2a70a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:46:49.506 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.435+0000 7f4d2a70a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:46:49.506 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.503+0000 7f4d2a70a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: [17/Sep/2024:01:46:47] ENGINE Bus STARTING 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: [17/Sep/2024:01:46:47] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: [17/Sep/2024:01:46:47] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: [17/Sep/2024:01:46:47] ENGINE Bus STARTED 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: [17/Sep/2024:01:46:47] ENGINE Error in HTTPServer.serve 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:46:49.594 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:46:49.595 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.do_handshake() 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[23696]: mgrmap e40: y(active, since 3s) 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: [17/Sep/2024:01:46:47] ENGINE Bus STARTING 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: [17/Sep/2024:01:46:47] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: [17/Sep/2024:01:46:47] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: [17/Sep/2024:01:46:47] ENGINE Bus STARTED 2024-09-17T01:46:49.596 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: [17/Sep/2024:01:46:47] ENGINE Error in HTTPServer.serve 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:46:49.597 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.do_handshake() 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:49.598 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:49 smithi081 ceph-mon[28146]: mgrmap e40: y(active, since 3s) 2024-09-17T01:46:49.807 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.565+0000 7f4d2a70a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:46:49.807 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.743+0000 7f4d2a70a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:46:50.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.808+0000 7f4d2a70a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:46:50.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.877+0000 7f4d2a70a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:46:50.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:49.961+0000 7f4d2a70a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:46:50.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:50.021+0000 7f4d2a70a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:46:50.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:50.081+0000 7f4d2a70a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:46:51.313 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:50.908+0000 7f4d2a70a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:46:51.313 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:46:50] ENGINE Bus STARTING 2024-09-17T01:46:51.313 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: CherryPy Checker: 2024-09-17T01:46:51.313 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: The Application mounted at '' has an empty config. 2024-09-17T01:46:51.313 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: 2024-09-17T01:46:51.314 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:46:51] ENGINE Serving on http://:::9283 2024-09-17T01:46:51.314 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: [17/Sep/2024:01:46:51] ENGINE Bus STARTED 2024-09-17T01:46:51.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:51.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: mgrmap e41: y(active, since 5s) 2024-09-17T01:46:51.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:46:51.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: mgrmap e41: y(active, since 5s) 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:46:51.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:51 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: mgrmap e41: y(active, since 5s) 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:46:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:46:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2907546554' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:46:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:51 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: mgrmap e42: y(active, since 6s), standbys: x 2024-09-17T01:46:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:46:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:52.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:52 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: mgrmap e42: y(active, since 6s), standbys: x 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: mgrmap e42: y(active, since 6s), standbys: x 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:46:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:52 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:46:53.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[44251]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:46:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-17T01:46:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-17T01:46:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:53 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-17T01:46:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:46:53.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:46:53.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:53.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:53 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 0 B/s wr, 228 op/s 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:46:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:54 smithi105 ceph-mon[24602]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:46:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 0 B/s wr, 228 op/s 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[28146]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:46:54.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 0 B/s wr, 228 op/s 2024-09-17T01:46:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T01:46:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Reconfiguring daemon prometheus.a on smithi105 2024-09-17T01:46:54.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:54 smithi081 ceph-mon[23696]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:46:56.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-mon[24602]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 0 B/s wr, 171 op/s 2024-09-17T01:46:56.386 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:46:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:56 smithi081 ceph-mon[23696]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 0 B/s wr, 171 op/s 2024-09-17T01:46:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:56 smithi081 ceph-mon[28146]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 120 KiB/s rd, 0 B/s wr, 171 op/s 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.576Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T01:46:56.867 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-17T01:46:56.868 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-17T01:46:56.868 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T01:46:56.868 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-17T01:46:56.868 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[56224]: ts=2024-09-17T01:46:56.577Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-17T01:46:56.868 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 podman[58976]: 2024-09-17 01:46:56.58759778 +0000 UTC m=+0.112983793 container died 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:57.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 podman[58976]: 2024-09-17 01:46:56.985726677 +0000 UTC m=+0.511112672 container cleanup 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:57.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:56 smithi105 bash[58976]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T01:46:57.402 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 podman[58987]: 2024-09-17 01:46:57.153407942 +0000 UTC m=+0.567555806 container remove 65d14fae06580978df1e61d0d2688a4d845ab6b9fdc6985bbb29c37589cee46f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 systemd[1]: Stopped Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.737s CPU time. 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 systemd[1]: Starting Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 podman[59082]: 2024-09-17 01:46:57.521776219 +0000 UTC m=+0.017350709 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-17T01:46:57.750 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 podman[59082]: 2024-09-17 01:46:57.629455443 +0000 UTC m=+0.125029926 container create 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:58.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 podman[59082]: 2024-09-17 01:46:57.856658948 +0000 UTC m=+0.352233426 container init 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:58.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 podman[59082]: 2024-09-17 01:46:57.859632267 +0000 UTC m=+0.355206751 container start 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T01:46:58.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.883Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-17T01:46:58.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.883Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-17T01:46:58.136 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.883Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.883Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.883Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.884Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.885Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.886Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.886Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.887Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.887Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.462µs 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.887Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T01:46:58.137 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.894Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.909Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.922Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.925Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.927Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.927Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.927Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.539µs wal_replay_duration=39.603209ms wbl_replay_duration=90ns total_replay_duration=39.650382ms 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.930Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.930Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.930Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.950Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.111235ms db_storage=850ns remote_storage=1.026µs web_handler=335ns query_engine=761ns scrape=364.2µs scrape_sd=92.226µs notify=16.259µs notify_sd=16.396µs rules=19.221384ms tracing=7.049µs 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.950Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:46:57.950Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-17T01:46:58.138 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 bash[59082]: 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa 2024-09-17T01:46:58.139 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:46:57 smithi105 systemd[1]: Started Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:46:58.588 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:58 smithi105 ceph-mon[24602]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-09-17T01:46:58.588 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:58 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.588 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:58 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.588 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:58 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:46:58.588 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:58 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[23696]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-09-17T01:46:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[28146]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 155 op/s 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:46:58.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:58 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:46:59.378 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:46:59 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:46:59.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:46:59 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:46:59.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:46:59 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:47:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:00 smithi105 ceph-mon[24602]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:00 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:00 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:00 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:47:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:00 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[28146]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:47:00.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[23696]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:47:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:47:00.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:00 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:01.295 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:00 smithi105 systemd[1]: Stopping Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:47:01.581 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:01 smithi105 podman[59432]: 2024-09-17 01:47:01.298114007 +0000 UTC m=+0.156059333 container died d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:47:01.582 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:01 smithi105 ceph-mon[24602]: Upgrade: Updating mgr.x 2024-09-17T01:47:01.582 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:01 smithi105 ceph-mon[24602]: Deploying daemon mgr.x on smithi105 2024-09-17T01:47:01.582 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:01 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:01.582 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:01 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[23696]: Upgrade: Updating mgr.x 2024-09-17T01:47:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[23696]: Deploying daemon mgr.x on smithi105 2024-09-17T01:47:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[28146]: Upgrade: Updating mgr.x 2024-09-17T01:47:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[28146]: Deploying daemon mgr.x on smithi105 2024-09-17T01:47:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:01.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:01 smithi105 podman[59432]: 2024-09-17 01:47:01.70062567 +0000 UTC m=+0.558570995 container cleanup d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T01:47:01.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:01 smithi105 bash[59432]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:01 smithi105 podman[59444]: 2024-09-17 01:47:01.968890675 +0000 UTC m=+0.668388584 container remove d4a850b74b11d7b87a814af0a42b4f373879fabfcf8f75e38fb8b399fda2b54c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:01 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 systemd[1]: Stopped Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Consumed 32.398s CPU time. 2024-09-17T01:47:02.302 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 systemd[1]: Starting Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:47:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:02 smithi105 ceph-mon[24602]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:02 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:02.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 podman[59574]: 2024-09-17 01:47:02.417445865 +0000 UTC m=+0.015784070 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:47:02.636 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 podman[59574]: 2024-09-17 01:47:02.528225168 +0000 UTC m=+0.126563408 container create a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:47:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:02 smithi081 ceph-mon[23696]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:02.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:02.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:02 smithi081 ceph-mon[28146]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-09-17T01:47:02.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:02.945 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 podman[59574]: 2024-09-17 01:47:02.755524828 +0000 UTC m=+0.353863027 container init a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:47:02.945 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 podman[59574]: 2024-09-17 01:47:02.758392717 +0000 UTC m=+0.356730920 container start a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97) 2024-09-17T01:47:02.945 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 ceph-mgr[59592]: -- 172.21.15.105:0/4041121880 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cf923b70e0 con 0x55cf92394800 2024-09-17T01:47:02.946 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 bash[59574]: a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 2024-09-17T01:47:02.946 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 systemd[1]: Started Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:47:02.946 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:02.883+0000 7fcaae77c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:47:02.946 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:02.947+0000 7fcaae77c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:47:03.334 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.102+0000 7fcaae77c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:47:03.334 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.171+0000 7fcaae77c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:47:03.334 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.336+0000 7fcaae77c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:47:03.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.477+0000 7fcaae77c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:47:03.739 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:03] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.43.0" 2024-09-17T01:47:04.063 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-mon[24602]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 121 op/s 2024-09-17T01:47:04.063 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.063 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.064 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.808+0000 7fcaae77c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:47:04.064 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:03.870+0000 7fcaae77c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:47:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[23696]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 121 op/s 2024-09-17T01:47:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[28146]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 0 B/s wr, 121 op/s 2024-09-17T01:47:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:03 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:04.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:04.064+0000 7fcaae77c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:47:05.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:05.017+0000 7fcaae77c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:47:05.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:05.084+0000 7fcaae77c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:47:06.092 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:05.823+0000 7fcaae77c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:47:06.092 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:05 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:05.950+0000 7fcaae77c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:47:06.384 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-mon[24602]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-17T01:47:06.384 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.094+0000 7fcaae77c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:47:06.384 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.155+0000 7fcaae77c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:47:06.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.311+0000 7fcaae77c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:47:06.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.386+0000 7fcaae77c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:47:06.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.446+0000 7fcaae77c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:47:06.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.611+0000 7fcaae77c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:47:06.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:06 smithi081 ceph-mon[23696]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-17T01:47:06.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:06 smithi081 ceph-mon[28146]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 0 B/s wr, 15 op/s 2024-09-17T01:47:06.955 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.682+0000 7fcaae77c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:47:06.955 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.750+0000 7fcaae77c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:47:06.955 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.835+0000 7fcaae77c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:47:06.955 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.896+0000 7fcaae77c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:47:06.955 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:06 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:06.957+0000 7fcaae77c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:07.805+0000 7fcaae77c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:47:07] ENGINE Bus STARTING 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: CherryPy Checker: 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: The Application mounted at '' has an empty config. 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:47:07] ENGINE Serving on http://:::9283 2024-09-17T01:47:08.059 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:47:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:47:07] ENGINE Bus STARTED 2024-09-17T01:47:08.335 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T01:47:08.335 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.335 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: Standby manager daemon x restarted 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: Standby manager daemon x started 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:47:08.336 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:08 smithi105 ceph-mon[24602]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:47:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T01:47:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: Standby manager daemon x restarted 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: Standby manager daemon x started 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[23696]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:47:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: Standby manager daemon x restarted 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:47:08.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:08 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/2361232436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:47:10.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:09 smithi105 ceph-mon[24602]: mgrmap e43: y(active, since 23s), standbys: x 2024-09-17T01:47:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:09 smithi105 ceph-mon[24602]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:10.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:09 smithi081 ceph-mon[23696]: mgrmap e43: y(active, since 23s), standbys: x 2024-09-17T01:47:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:09 smithi081 ceph-mon[23696]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:09 smithi081 ceph-mon[28146]: mgrmap e43: y(active, since 23s), standbys: x 2024-09-17T01:47:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:09 smithi081 ceph-mon[28146]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:12.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:12.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:12.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all mgr 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all crash 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all mds 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all nfs 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:47:12.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:11 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:47:12.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:12.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:12.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:12.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:47:12.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:12.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all mgr 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T01:47:12.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T01:47:12.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all crash 2024-09-17T01:47:12.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all mds 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.186 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all nfs 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:47:12.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:47:12.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:47:12.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:12.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:12.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:12.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:47:12.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:12.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.191 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:12.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:12.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all mgr 2024-09-17T01:47:12.192 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T01:47:12.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T01:47:12.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all crash 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all mds 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all nfs 2024-09-17T01:47:12.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:47:12.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:12.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:47:12.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:47:12.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:12.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:11 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:47:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:13] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.43.0" 2024-09-17T01:47:14.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:13 smithi105 ceph-mon[24602]: Upgrade: Updating grafana.a 2024-09-17T01:47:14.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:13 smithi105 ceph-mon[24602]: Deploying daemon grafana.a on smithi105 2024-09-17T01:47:14.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:13 smithi105 ceph-mon[24602]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[28146]: Upgrade: Updating grafana.a 2024-09-17T01:47:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[28146]: Deploying daemon grafana.a on smithi105 2024-09-17T01:47:14.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[28146]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[23696]: Upgrade: Updating grafana.a 2024-09-17T01:47:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[23696]: Deploying daemon grafana.a on smithi105 2024-09-17T01:47:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:13 smithi081 ceph-mon[23696]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:15.714 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (13m) 25s ago 21m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (12m) 9s ago 20m 53.3M - 8.3.5 dad864ee21e9 df0f90cfa3fb 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 unknown 25s ago 20m 50.9M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (13s) 9s ago 25m 69.2M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (9m) 25s ago 27m 475M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (27m) 25s ago 27m 75.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:47:16.362 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (25m) 9s ago 25m 58.5M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (26m) 25s ago 26m 54.4M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (13m) 25s ago 21m 14.9M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (12m) 9s ago 21m 15.8M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (25m) 25s ago 25m 53.9M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (24m) 25s ago 24m 54.8M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (24m) 25s ago 24m 50.2M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (24m) 25s ago 24m 53.7M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (23m) 9s ago 23m 55.6M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (23m) 9s ago 23m 51.9M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (23m) 9s ago 23m 53.0M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (22m) 9s ago 22m 54.6M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (18s) 9s ago 22m 47.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (20m) 25s ago 20m 77.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:47:16.363 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (20m) 9s ago 20m 78.8M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:47:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:16 smithi105 ceph-mon[24602]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:16.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:16 smithi081 ceph-mon[23696]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:16.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:16 smithi081 ceph-mon[28146]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:47:16.756 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:47:16.757 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:47:17.086 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [ 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr" 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:47:17.087 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:47:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[23696]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[23696]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2214956733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[28146]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[28146]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:17 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2214956733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:17 smithi105 ceph-mon[24602]: from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:17 smithi105 ceph-mon[24602]: from='client.25348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:17 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:17.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:17 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/2214956733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:18 smithi105 ceph-mon[24602]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:18 smithi105 ceph-mon[24602]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:18 smithi105 ceph-mon[24602]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[23696]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[23696]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[23696]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:18.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[28146]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[28146]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:18 smithi081 ceph-mon[28146]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:20.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:20 smithi105 ceph-mon[24602]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:20.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:20 smithi081 ceph-mon[23696]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:20.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:20 smithi081 ceph-mon[28146]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:22.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:21 smithi105 systemd[1]: Stopping Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:47:22.322 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:22 smithi105 ceph-mon[24602]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:22.322 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:22 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:22.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[48220]: t=2024-09-17T01:47:22+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-17T01:47:22.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 podman[61146]: 2024-09-17 01:47:22.067460795 +0000 UTC m=+0.130465503 container died df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, vendor=Red Hat, Inc., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git) 2024-09-17T01:47:22.598 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:22 smithi081 ceph-mon[23696]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:22.598 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:22 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:22.599 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:22 smithi081 ceph-mon[28146]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:22.599 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:22 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:22.635 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 podman[61146]: 2024-09-17 01:47:22.423546366 +0000 UTC m=+0.486551075 container cleanup df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, com.redhat.component=ubi8-container, distribution-scope=public, release=236.1648460182, maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.buildah.version=1.24.2, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762) 2024-09-17T01:47:22.636 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 bash[61146]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a 2024-09-17T01:47:23.043 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 podman[61158]: 2024-09-17 01:47:22.691472568 +0000 UTC m=+0.625310142 container remove df0f90cfa3fba00cb5adb0083f0b4818707c9f0e1c9ab8ef77aab007a75c8325 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, version=8.5, distribution-scope=public, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, name=ubi8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-09-17T01:47:23.043 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-17T01:47:23.043 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 systemd[1]: Stopped Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:47:23.043 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:22 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Consumed 4.961s CPU time. 2024-09-17T01:47:23.043 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 systemd[1]: Starting Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:47:23.385 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 podman[61286]: 2024-09-17 01:47:23.155003821 +0000 UTC m=+0.016589538 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-09-17T01:47:23.385 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 podman[61286]: 2024-09-17 01:47:23.259350988 +0000 UTC m=+0.120936689 container create 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T01:47:23.676 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 podman[61286]: 2024-09-17 01:47:23.520395612 +0000 UTC m=+0.381981320 container init 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T01:47:23.676 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 podman[61286]: 2024-09-17 01:47:23.523348289 +0000 UTC m=+0.384933997 container start 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T01:47:23.676 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 bash[61286]: 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a 2024-09-17T01:47:23.677 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 systemd[1]: Started Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:47:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:23] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-09-17T01:47:24.136 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679195621Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-09-17T01:47:24.136 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679382046Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-17T01:47:24.136 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679398233Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-17T01:47:24.136 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679403358Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-17T01:47:24.136 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679407897Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.67941257Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679416813Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.6794205Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679424136Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679428365Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679432604Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679436795Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679441356Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679445997Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679450425Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679454605Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-17T01:47:24.137 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679458739Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-17T01:47:24.138 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679463295Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-17T01:47:24.138 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=settings t=2024-09-17T01:47:23.679467654Z level=info msg="App mode production" 2024-09-17T01:47:24.138 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=sqlstore t=2024-09-17T01:47:23.679525362Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-17T01:47:24.138 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=sqlstore t=2024-09-17T01:47:23.679542622Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-17T01:47:24.138 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.692904012Z level=info msg="Starting DB migrations" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.696778817Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.708590926Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.709765438Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.710596632Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.711866785Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.713117754Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-17T01:47:24.139 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.714195679Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.715055147Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.715576419Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.716105225Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.716861364Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.717923729Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.718248634Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-17T01:47:24.140 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.719373501Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.720446064Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.721669116Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.722047759Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.723309052Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.724538156Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-17T01:47:24.141 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.724845499Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.726087155Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.727173201Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.727654912Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.72843477Z level=info msg="Executing migration" id="create alert_image table" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.729246818Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.730049868Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-17T01:47:24.142 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.730363283Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.730798494Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.73123642Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.731309435Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.731537401Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.732023572Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-17T01:47:24.143 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.732496093Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.732767226Z level=info msg="Executing migration" id="create secrets table" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.733265466Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.740325251Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.741554559Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.741870104Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-17T01:47:24.144 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.748333732Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-17T01:47:24.145 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.754610121Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-17T01:47:24.145 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.755751221Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-17T01:47:24.145 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.756526154Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-17T01:47:24.145 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.75705395Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-17T01:47:24.146 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.757333298Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-17T01:47:24.146 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.757758168Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-17T01:47:24.146 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.759055843Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-17T01:47:24.146 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.759832019Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-17T01:47:24.146 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.760146504Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-17T01:47:24.147 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.760458229Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-17T01:47:24.147 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.760829652Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-17T01:47:24.147 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.761263607Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-17T01:47:24.147 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.76236304Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-17T01:47:24.148 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.762671129Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-17T01:47:24.148 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.763093698Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-17T01:47:24.149 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.76350838Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-17T01:47:24.149 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.76395251Z level=info msg="Executing migration" id="add correlation config column" 2024-09-17T01:47:24.149 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.765076544Z level=info msg="Executing migration" id="create entity_events table" 2024-09-17T01:47:24.149 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.76545616Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-17T01:47:24.149 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.765853299Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.765931556Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.766197336Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.76626884Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.766534349Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.766960571Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-17T01:47:24.150 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.767820236Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.768675923Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.769513737Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.770361563Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.771198254Z level=info msg="Executing migration" id="Drop public config table" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.772057389Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-17T01:47:24.151 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.772904379Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.773751402Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.774625189Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.775481894Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.779392387Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.780567965Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-17T01:47:24.152 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.781788815Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.782664901Z level=info msg="Executing migration" id="add share column" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.783873022Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.784731272Z level=info msg="Executing migration" id="create file table" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.785610909Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.786441159Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-17T01:47:24.153 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.7873039Z level=info msg="Executing migration" id="create file_meta table" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.78815893Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.788971246Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.791798279Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.794759501Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.79750336Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-17T01:47:24.154 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.800179462Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.802997957Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.805843836Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.808625062Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.811540565Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.814185533Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.816997631Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-17T01:47:24.155 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.818250029Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.819464022Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.820339297Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.833559395Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.834410553Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.835246197Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-17T01:47:24.156 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.839706996Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.840576209Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.841410473Z level=info msg="Executing migration" id="create folder table" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.842281859Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.843131849Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.843956704Z level=info msg="Executing migration" id="Update folder title length" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.844804641Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=migrator t=2024-09-17T01:47:23.845641822Z level=info msg="migrations completed" performed=105 skipped=376 duration=148.914149ms 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=sqlstore t=2024-09-17T01:47:23.846031636Z level=info msg="Created default organization" 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=plugin.finder t=2024-09-17T01:47:23.879025133Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=secrets t=2024-09-17T01:47:23.879150621Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-17T01:47:24.157 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=query_data t=2024-09-17T01:47:23.881346898Z level=info msg="Query Service initialization" 2024-09-17T01:47:24.158 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:23 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=live.push_http t=2024-09-17T01:47:23.884114698Z level=info msg="Live Push Gateway initialization" 2024-09-17T01:47:24.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-mon[24602]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:24.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=infra.usagestats.collector t=2024-09-17T01:47:24.521616974Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=provisioning.datasources t=2024-09-17T01:47:24.523620357Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=provisioning.datasources t=2024-09-17T01:47:24.523628747Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=provisioning.datasources t=2024-09-17T01:47:24.523723421Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=provisioning.alerting t=2024-09-17T01:47:24.545768333Z level=info msg="starting to provision alerting" 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=provisioning.alerting t=2024-09-17T01:47:24.545778622Z level=info msg="finished to provision alerting" 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=grafanaStorageLogger t=2024-09-17T01:47:24.545958439Z level=info msg="storage starting" 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=ngalert.state.manager t=2024-09-17T01:47:24.546014182Z level=info msg="Warming state cache for startup" 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=http.server t=2024-09-17T01:47:24.548001789Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=ngalert.state.manager t=2024-09-17T01:47:24.576248041Z level=info msg="State cache has been initialized" states=0 duration=30.232381ms 2024-09-17T01:47:24.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=ticker t=2024-09-17T01:47:24.576301693Z level=info msg=starting first_tick=2024-09-17T01:47:30Z 2024-09-17T01:47:24.575 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=ngalert.multiorg.alertmanager t=2024-09-17T01:47:24.57630289Z level=info msg="starting MultiOrg Alertmanager" 2024-09-17T01:47:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[23696]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:24.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[28146]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:24.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:24 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:24.885 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=sqlstore.transactions t=2024-09-17T01:47:24.587763654Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-17T01:47:24.885 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:47:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=sqlstore.transactions t=2024-09-17T01:47:24.599262729Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-17T01:47:26.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:26 smithi105 ceph-mon[24602]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:26 smithi081 ceph-mon[23696]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:26.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:26 smithi081 ceph-mon[28146]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:28 smithi105 ceph-mon[24602]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:28 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:28 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[23696]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:28.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[28146]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:28.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:28 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:29 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:29 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:29 smithi105 ceph-mon[24602]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:29.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[23696]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:29.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:29.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:29 smithi081 ceph-mon[28146]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[23696]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[28146]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:32 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:32.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:32 smithi105 ceph-mon[24602]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:32.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:32 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:32.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:32 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:33.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: Upgrade: Finalizing container_image settings 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: Upgrade: Complete! 2024-09-17T01:47:33.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:33 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:47:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:33] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: Upgrade: Finalizing container_image settings 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: Upgrade: Finalizing container_image settings 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:47:33.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:47:33.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:47:33.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: Upgrade: Complete! 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:47:33.941 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:47:33.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: Upgrade: Complete! 2024-09-17T01:47:33.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:47:33.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:33 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:47:36.515 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:36 smithi105 ceph-mon[24602]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:36.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:36 smithi081 ceph-mon[23696]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:36.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:36 smithi081 ceph-mon[28146]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:47:37.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:37 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:37 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:37 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 526 B/s rd, 0 op/s 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:38 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 526 B/s rd, 0 op/s 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 526 B/s rd, 0 op/s 2024-09-17T01:47:38.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:47:38.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:38 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:47:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:39 smithi105 ceph-mon[24602]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:47:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:39 smithi105 ceph-mon[24602]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:47:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:39 smithi105 ceph-mon[24602]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:39 smithi105 ceph-mon[24602]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[23696]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:47:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[23696]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[23696]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[23696]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[28146]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[28146]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[28146]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:39.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:39 smithi081 ceph-mon[28146]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:41 smithi105 ceph-mon[24602]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:41 smithi081 ceph-mon[23696]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:41 smithi081 ceph-mon[28146]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 902 B/s rd, 0 op/s 2024-09-17T01:47:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:42 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:42 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:42.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:42 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:43 smithi105 ceph-mon[24602]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-17T01:47:43.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:43 smithi081 ceph-mon[23696]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-17T01:47:43.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:43 smithi081 ceph-mon[28146]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-09-17T01:47:43.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:43] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-09-17T01:47:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:45 smithi105 ceph-mon[24602]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-09-17T01:47:45.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:45 smithi081 ceph-mon[23696]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-09-17T01:47:45.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:45 smithi081 ceph-mon[28146]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-09-17T01:47:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:46 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:46.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:46.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:47:47.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:47 smithi105 ceph-mon[24602]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-17T01:47:47.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:47 smithi081 ceph-mon[23696]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-17T01:47:47.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:47 smithi081 ceph-mon[28146]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-09-17T01:47:48.041 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-17T01:47:48.660 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:48 smithi081 ceph-mon[23696]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:48.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:48 smithi081 ceph-mon[23696]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:48.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:48 smithi081 ceph-mon[28146]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:48.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:48 smithi081 ceph-mon[28146]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:48 smithi105 ceph-mon[24602]: from='client.25372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:48.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:48 smithi105 ceph-mon[24602]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:47:50.093 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:47:50.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:50 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/952139540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:50.395 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:50 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/952139540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:50.395 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:50 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/952139540' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:50.677 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-09-17T01:47:51.603 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:51 smithi081 ceph-mon[23696]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:51.603 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:51 smithi081 ceph-mon[28146]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:51 smithi105 ceph-mon[24602]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:52 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:52 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:52 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:47:52.725 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:47:53.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:53 smithi081 ceph-mon[23696]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:53.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:53 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/24696021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:53.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:53 smithi081 ceph-mon[28146]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:53.394 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:53 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/24696021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:53.416 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-09-17T01:47:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:53 smithi105 ceph-mon[24602]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:53 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/24696021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:53.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:47:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:47:53] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-09-17T01:47:55.455 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:47:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:55 smithi105 ceph-mon[24602]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:55 smithi081 ceph-mon[23696]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:55.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:55 smithi081 ceph-mon[28146]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:55.937 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-09-17T01:47:56.568 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:56 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/54019873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:56.568 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:56 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/54019873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:56 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/54019873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:47:57.499 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:57 smithi081 ceph-mon[23696]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:57.499 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:57 smithi081 ceph-mon[28146]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:57.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:57 smithi105 ceph-mon[24602]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:47:59.172 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:47:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:59 smithi081 ceph-mon[23696]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:47:59 smithi081 ceph-mon[23696]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:59 smithi081 ceph-mon[28146]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:47:59 smithi081 ceph-mon[28146]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:59 smithi105 ceph-mon[24602]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:47:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:47:59 smithi105 ceph-mon[24602]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:47:59.800 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:48:01.327 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:01 smithi081 ceph-mon[23696]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:01 smithi105 ceph-mon[24602]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:01 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:01.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:01 smithi081 ceph-mon[28146]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:01.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:48:01.790 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:48:01.791 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:48:01.791 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:48:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:02.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:02.497 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:48:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:02 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:03.631 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:03 smithi081 ceph-mon[23696]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:03.631 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:03 smithi081 ceph-mon[23696]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:03.631 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:03 smithi081 ceph-mon[28146]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:03.631 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:03 smithi081 ceph-mon[28146]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:03.631 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:03] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-09-17T01:48:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:03 smithi105 ceph-mon[24602]: from='client.15498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:03 smithi105 ceph-mon[24602]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:04.411 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:48:04.411 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:48:04.411 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:48:04.971 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-09-17T01:48:05.526 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:05 smithi081 ceph-mon[23696]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:05.526 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:05 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3455642821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:05.527 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:05 smithi081 ceph-mon[28146]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:05.527 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:05 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3455642821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:05 smithi105 ceph-mon[24602]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:05 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3455642821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:07 smithi105 ceph-mon[24602]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:07.672 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:07 smithi081 ceph-mon[23696]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:07.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:07 smithi081 ceph-mon[28146]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:08.607 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:08.607 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:08 smithi081 ceph-mon[23696]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:08.607 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:08 smithi081 ceph-mon[23696]: from='client.25420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:08.607 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:08 smithi081 ceph-mon[28146]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:08.607 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:08 smithi081 ceph-mon[28146]: from='client.25420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:08 smithi105 ceph-mon[24602]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:08 smithi105 ceph-mon[24602]: from='client.25420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:09.277 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:48:09.585 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:09 smithi081 ceph-mon[23696]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:09.585 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:09 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:09.585 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:09 smithi081 ceph-mon[28146]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:09.586 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:09 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:09 smithi105 ceph-mon[24602]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:09.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:09 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:10 smithi105 ceph-mon[24602]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:10 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:10 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:10.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:10 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[23696]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[28146]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:10.676 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:10 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:11.407 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:48:11.527 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[23696]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-09-17T01:48:11.527 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[23696]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:11.527 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[23696]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:11.528 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[28146]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-09-17T01:48:11.528 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[28146]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:11.528 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:11 smithi081 ceph-mon[28146]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:11 smithi105 ceph-mon[24602]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 922 B/s rd, 0 op/s 2024-09-17T01:48:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:11 smithi105 ceph-mon[24602]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:11 smithi105 ceph-mon[24602]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (14m) 80s ago 22m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (48s) 33s ago 21m 73.7M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 unknown 80s ago 21m 50.9M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (69s) 33s ago 26m 439M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:48:12.073 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (10m) 80s ago 28m 475M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (28m) 80s ago 28m 75.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (26m) 33s ago 26m 58.5M 2048M 17.2.0 e1d6a67b021e 50fdcb95e6c5 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (26m) 80s ago 26m 54.4M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (14m) 80s ago 22m 14.9M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (13m) 33s ago 22m 15.6M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (26m) 80s ago 26m 53.9M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (25m) 80s ago 25m 54.8M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (25m) 80s ago 25m 50.2M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (25m) 80s ago 25m 53.7M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (24m) 33s ago 24m 55.9M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (24m) 33s ago 24m 51.9M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (23m) 33s ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:48:12.074 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (23m) 33s ago 23m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:48:12.075 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (74s) 33s ago 22m 53.3M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:48:12.075 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (21m) 80s ago 21m 77.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:48:12.075 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (21m) 33s ago 21m 79.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:48:12.472 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:48:12.472 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:48:12.472 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-09-17T01:48:12.472 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:48:12.473 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:48:12.474 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:48:12.474 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:48:12.474 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:48:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:12 smithi105 ceph-mon[24602]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:12 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:12 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:12.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:12 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:12 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi105", 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "0/1 daemons upgraded", 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:48:12.806 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:48:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: Upgrade: Setting container_image for all mgr 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: Upgrade: It appears safe to stop mon.b 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='client.? 172.21.15.81:0/3422264558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:13.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:13 smithi105 ceph-mon[24602]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:13] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all mgr 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: Upgrade: It appears safe to stop mon.b 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3422264558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:13.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[23696]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='client.15528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='client.15534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all mgr 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T01:48:13.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: Upgrade: It appears safe to stop mon.b 2024-09-17T01:48:13.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3422264558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:13.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:13 smithi081 ceph-mon[28146]: from='client.15546 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: Upgrade: Updating mon.b 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:48:15.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:15.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:14 smithi105 ceph-mon[24602]: Deploying daemon mon.b on smithi105 2024-09-17T01:48:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: Upgrade: Updating mon.b 2024-09-17T01:48:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[23696]: Deploying daemon mon.b on smithi105 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: Upgrade: Updating mon.b 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:48:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:15.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:14 smithi081 ceph-mon[28146]: Deploying daemon mon.b on smithi105 2024-09-17T01:48:15.566 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:15 smithi105 systemd[1]: Stopping Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:48:15.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:15 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b[24585]: 2024-09-17T01:48:15.567+0000 7f080f39f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:48:15.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:15 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b[24585]: 2024-09-17T01:48:15.567+0000 7f080f39f700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-17T01:48:15.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:15 smithi105 podman[63266]: 2024-09-17 01:48:15.669495158 +0000 UTC m=+0.208620173 container died 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-type=git, io.openshift.expose-services=, RELEASE=HEAD, GIT_BRANCH=HEAD, name=centos-stream, GIT_CLEAN=True) 2024-09-17T01:48:16.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 podman[63266]: 2024-09-17 01:48:16.042337353 +0000 UTC m=+0.581462396 container cleanup 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-09-17T01:48:16.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 bash[63266]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b 2024-09-17T01:48:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 podman[63279]: 2024-09-17 01:48:16.310239847 +0000 UTC m=+0.643411429 container remove 50fdcb95e6c57a64338b167140dee51bb615297542c2b7fc6cea7a715dd44502 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T01:48:16.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-17T01:48:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 systemd[1]: Stopped Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:48:16.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.b.service: Consumed 16.139s CPU time. 2024-09-17T01:48:16.982 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 systemd[1]: Starting Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:48:16.982 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 podman[63408]: 2024-09-17 01:48:16.761486994 +0000 UTC m=+0.015989993 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:16.982 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:16 smithi105 podman[63408]: 2024-09-17 01:48:16.861121535 +0000 UTC m=+0.115624528 container create 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T01:48:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 podman[63408]: 2024-09-17 01:48:17.105512412 +0000 UTC m=+0.360015405 container init 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 podman[63408]: 2024-09-17 01:48:17.109020472 +0000 UTC m=+0.363523476 container start 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: pidfile_write: ignore empty --pid-file 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: load: jerasure load: lrc 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: RocksDB version: 7.9.2 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Git sha 0 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: DB SUMMARY 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: DB Session ID: RAG8SG2D87FVYIHDKQEQ 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:48:17.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2829 Bytes 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 599251 ; 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.env: 0x561ecab23940 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T01:48:17.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.info_log: 0x561ecb9956e0 2024-09-17T01:48:17.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:48:17.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.statistics: (nil) 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.use_fsync: 0 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:48:17.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.db_log_dir: 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.wal_dir: 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:48:17.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.write_buffer_manager: 0x561ecb9a48c0 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:48:17.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.unordered_write: 0 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.row_cache: None 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.wal_filter: None 2024-09-17T01:48:17.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.wal_compression: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:48:17.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:48:17.394 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_open_files: -1 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Compression algorithms supported: 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kZSTD supported: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:48:17.395 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.merge_operator: 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_filter: None 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:48:17.396 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561ecb995320) 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_type: 0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_index_type: 0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_shortening: 1 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: checksum: 4 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: no_block_cache: 0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache: 0x561ecb9891f0 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_options: 2024-09-17T01:48:17.397 INFO:journalctl@ceph.mon.b.smithi105.stdout: capacity : 536870912 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_shard_bits : 4 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: strict_capacity_limit : 0 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_compressed: (nil) 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: persistent_cache: (nil) 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size: 4096 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size_deviation: 10 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_restart_interval: 16 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_block_restart_interval: 1 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: metadata_block_size: 4096 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: partition_filters: 0 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: use_delta_encoding: 1 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: filter_policy: bloomfilter 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: whole_key_filtering: 1 2024-09-17T01:48:17.398 INFO:journalctl@ceph.mon.b.smithi105.stdout: verify_compression: 0 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: format_version: 5 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_index_compression: 1 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_align: 0 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_auto_readahead_size: 262144 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: prepopulate_block_cache: 0 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: initial_auto_readahead_size: 8192 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression: NoCompression 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:48:17.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.num_levels: 7 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:48:17.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:48:17.401 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T01:48:17.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T01:48:17.403 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.ttl: 2592000 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:48:17.404 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-09-17T01:48:17.405 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28742, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e46a32cb-03d4-4d6a-b644-a22d423e195e 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537697135435, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537697137725, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 390358, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28747, "largest_seqno": 29232, "table_properties": {"data_size": 387879, "index_size": 927, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4264, "raw_average_key_size": 23, "raw_value_size": 384069, "raw_average_value_size": 2076, "num_data_blocks": 43, "num_entries": 185, "num_filter_entries": 185, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726537697, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e46a32cb-03d4-4d6a-b644-a22d423e195e", "db_session_id": "RAG8SG2D87FVYIHDKQEQ", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537697137830, "job": 1, "event": "recovery_finished"} 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561ecb9b6e00 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: DB pointer 0x561ecbab2000 2024-09-17T01:48:17.406 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** DB Stats ** 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T01:48:17.407 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 381.21 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: L6 1/0 8.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 2/0 8.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 286.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 286.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:48:17.408 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.00 GB write, 83.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.00 GB write, 83.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache BinnedLRUCache@0x561ecb9891f0#2 capacity: 512.00 MB usage: 1.67 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: starting mon.b rank 2 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???) e3 preinit fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:48:17.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).mds e1 new map 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).mds e1 print_map 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: e1 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: legacy client fscid: -1 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout: No filesystems configured 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 ceph-mon[63427]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-09-17T01:48:17.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 bash[63408]: 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec 2024-09-17T01:48:17.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:17 smithi105 systemd[1]: Started Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:48:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: mon.b calling monitor election 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: mon.c calling monitor election 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: mon.a calling monitor election 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: fsmap 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:18 smithi105 ceph-mon[63427]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: mon.b calling monitor election 2024-09-17T01:48:18.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: mon.c calling monitor election 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: mon.a calling monitor election 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: fsmap 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[23696]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: mon.b calling monitor election 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: mon.c calling monitor election 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: mon.a calling monitor election 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: monmap e3: 3 mons at {a=[v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0]} 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: fsmap 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: mgrmap e43: y(active, since 92s), standbys: x 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:48:18.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:48:18.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:18.683 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:18 smithi081 ceph-mon[28146]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:21.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:21 smithi105 ceph-mon[63427]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:21.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:21 smithi081 ceph-mon[23696]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:21.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:21 smithi081 ceph-mon[28146]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:22.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:22.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:22.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:22.431 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:48:22.148+0000 7f7f43b53640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-17T01:48:22.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:22.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:22.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:22 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:22 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:22 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:22.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:22 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:23 smithi105 ceph-mon[63427]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:23.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:23 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:23 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[23696]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:23] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[28146]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:23.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:23 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:25.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:25 smithi105 ceph-mon[63427]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:25.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:25 smithi081 ceph-mon[28146]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:25.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:25 smithi081 ceph-mon[23696]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:26.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Detected new or changed devices on smithi105 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all crash 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all mds 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.682 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all nfs 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:48:26.683 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Detected new or changed devices on smithi105 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.684 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all crash 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all mds 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.685 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all nfs 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.686 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all grafana 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all loki 2024-09-17T01:48:26.687 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all promtail 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Finalizing container_image settings 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:48:26.688 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all grafana 2024-09-17T01:48:26.689 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all loki 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all promtail 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Finalizing container_image settings 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.690 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.691 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.692 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.693 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.694 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.695 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: Upgrade: Complete! 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:48:26.696 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.697 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: Upgrade: Complete! 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.698 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.699 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.700 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:26 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Detected new or changed devices on smithi105 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:48:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mds 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nfs 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:48:26.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all grafana 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all loki 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all promtail 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:48:26.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 814 B/s rd, 0 op/s 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:26.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:26 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:29.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:28 smithi105 ceph-mon[63427]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:48:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:28 smithi081 ceph-mon[23696]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:48:29.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:28 smithi081 ceph-mon[28146]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:48:31.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:31 smithi105 ceph-mon[63427]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:31.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:31 smithi081 ceph-mon[23696]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:31.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:31 smithi081 ceph-mon[28146]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:48:32.569 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:32 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:32.569 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:32 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:32.569 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:32 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:32.570 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:32 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:32 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:32 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:33.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:33 smithi105 ceph-mon[63427]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:48:33.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:33 smithi081 ceph-mon[23696]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:48:33.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:33 smithi081 ceph-mon[28146]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:48:33.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:33] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-09-17T01:48:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:35 smithi105 ceph-mon[63427]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:35.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:35 smithi081 ceph-mon[23696]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:35.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:35 smithi081 ceph-mon[28146]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:37.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:37 smithi105 ceph-mon[63427]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:37.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:37 smithi081 ceph-mon[23696]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:37.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:37 smithi081 ceph-mon[28146]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:48:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:39 smithi105 ceph-mon[63427]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:39.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:39 smithi081 ceph-mon[23696]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:39.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:39 smithi081 ceph-mon[28146]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:41.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:41 smithi105 ceph-mon[63427]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:41.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:41 smithi081 ceph-mon[23696]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:41.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:41 smithi081 ceph-mon[28146]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:42 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:42.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:42 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:42.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:42 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:43 smithi081 ceph-mon[23696]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:43 smithi081 ceph-mon[28146]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:43 smithi105 ceph-mon[63427]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:43.906 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:43] "GET /metrics HTTP/1.1" 200 38003 "" "Prometheus/2.43.0" 2024-09-17T01:48:43.931 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:48:44.542 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:44 smithi081 ceph-mon[23696]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:44.543 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:44 smithi081 ceph-mon[28146]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:44.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:44 smithi105 ceph-mon[63427]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:45.446 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:45 smithi081 ceph-mon[23696]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:45.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:45 smithi081 ceph-mon[28146]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:45.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:45 smithi105 ceph-mon[63427]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:45.872 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:48:45.872 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (14m) 114s ago 22m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:48:45.872 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (82s) 24s ago 22m 76.9M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 unknown 114s ago 21m 50.9M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (103s) 24s ago 27m 440M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (11m) 114s ago 28m 475M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (28m) 114s ago 28m 75.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (29s) 24s ago 27m 18.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (27m) 114s ago 27m 54.4M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (14m) 114s ago 23m 14.9M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (14m) 24s ago 23m 15.5M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (26m) 114s ago 26m 53.9M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (26m) 114s ago 26m 54.8M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (26m) 114s ago 26m 50.2M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:48:45.873 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (25m) 114s ago 25m 53.7M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (25m) 24s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (24m) 24s ago 24m 52.2M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (24m) 24s ago 24m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (24m) 24s ago 24m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (108s) 24s ago 23m 61.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (21m) 114s ago 21m 77.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:48:45.874 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (21m) 24s ago 21m 79.8M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:48:46.503 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:46 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:46.503 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:46 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:46.526 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-09-17T01:48:46.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:46 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:48:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:47 smithi105 ceph-mon[63427]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:47.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:47 smithi105 ceph-mon[63427]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:47.668 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:47 smithi081 ceph-mon[28146]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:47.668 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:47 smithi081 ceph-mon[28146]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:47.669 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:47 smithi081 ceph-mon[23696]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:47.669 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:47 smithi081 ceph-mon[23696]: from='client.25462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:48.485 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:48:49.074 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:48:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:49 smithi105 ceph-mon[63427]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:49.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:49 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2515631228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:49.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:49 smithi081 ceph-mon[23696]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:49.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:49 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/2515631228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:49.679 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:49 smithi081 ceph-mon[28146]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:49.679 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:49 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/2515631228' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:48:51.046 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:48:51.047 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:48:51.047 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:48:51.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:51 smithi105 ceph-mon[63427]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:51.656 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:51 smithi081 ceph-mon[23696]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:51.656 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:51 smithi081 ceph-mon[28146]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:51.679 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:48:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:52 smithi105 ceph-mon[63427]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:52.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:52 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:52 smithi081 ceph-mon[23696]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:52.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:52 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:52 smithi081 ceph-mon[28146]: from='client.15570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:52.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:52 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:48:53.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:53 smithi081 ceph-mon[23696]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:53.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:53 smithi081 ceph-mon[28146]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:53.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:53 smithi105 ceph-mon[63427]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:53.691 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:48:53.691 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:48:53.691 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:48:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:48:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:48:53] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-09-17T01:48:54.361 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-09-17T01:48:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:54 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4055322357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:54.654 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:54 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/4055322357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:54.654 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:54 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/4055322357' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:48:55.626 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:55 smithi081 ceph-mon[23696]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:55.627 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:55 smithi081 ceph-mon[28146]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:55.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:55 smithi105 ceph-mon[63427]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:56 smithi105 ceph-mon[63427]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:56.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:56 smithi105 ceph-mon[63427]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:56 smithi081 ceph-mon[23696]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:56.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:56 smithi081 ceph-mon[23696]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:56.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:56 smithi081 ceph-mon[28146]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:48:56.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:56 smithi081 ceph-mon[28146]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:57.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:57 smithi105 ceph-mon[63427]: from='client.25492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi081", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:57 smithi081 ceph-mon[23696]: from='client.25492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi081", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:57.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:57 smithi081 ceph-mon[28146]: from='client.25492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "mon", "hosts": "smithi081", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:48:57.988 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:58.687 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:48:59.236 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[23696]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:59.237 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.238 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:48:58 smithi081 ceph-mon[28146]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:48:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:48:58 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:49:01.407 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:01.407 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:01.408 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:01.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:01 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-17T01:49:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:01.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:01 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-17T01:49:01.868 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:49:02.423 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: Upgrade: Setting container_image for all mgr 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: Upgrade: It appears safe to stop mon.a 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: Upgrade: Setting container_image for all mgr 2024-09-17T01:49:02.424 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: Upgrade: It appears safe to stop mon.a 2024-09-17T01:49:02.425 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:02.425 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:02 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:02.544 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (14m) 2m ago 22m 24.5M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (99s) 40s ago 22m 76.9M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 unknown 2m ago 22m 50.9M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (2m) 40s ago 27m 440M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (11m) 2m ago 28m 475M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (29m) 2m ago 29m 75.8M 2048M 17.2.0 e1d6a67b021e b8a531e37111 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (45s) 40s ago 27m 18.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (27m) 2m ago 27m 54.4M 2048M 17.2.0 e1d6a67b021e 4f17337c12f6 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (15m) 2m ago 23m 14.9M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (14m) 40s ago 23m 15.5M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (27m) 2m ago 27m 53.9M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (26m) 2m ago 26m 54.8M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (26m) 2m ago 26m 50.2M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:49:02.545 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (25m) 2m ago 25m 53.7M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (25m) 40s ago 25m 55.9M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (25m) 40s ago 25m 52.2M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (24m) 40s ago 24m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (24m) 40s ago 24m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (2m) 40s ago 23m 61.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (22m) 2m ago 22m 77.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:49:02.546 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (22m) 40s ago 22m 79.8M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:49:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:49:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:49:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:49:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: Upgrade: It appears safe to stop mon.a 2024-09-17T01:49:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:02 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:02.957 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:49:02.957 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:49:02.957 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T01:49:02.957 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:49:02.957 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "mds": {}, 2024-09-17T01:49:02.958 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:49:02.959 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:49:03.294 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:49:03.294 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:49:03.294 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:49:03.294 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi081", 2024-09-17T01:49:03.294 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:49:03.295 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "0/2 daemons upgraded", 2024-09-17T01:49:03.295 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T01:49:03.295 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:49:03.295 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:49:03.481 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[23696]: from='client.? 172.21.15.81:0/3894423717' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.482 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:03.483 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-mon[28146]: from='client.? 172.21.15.81:0/3894423717' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='mgr.y' 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='mgr.15267 172.21.15.81:0/1871768733' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:03.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:03 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3894423717' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:03.823 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:49:03] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-09-17T01:49:04.370 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 systemd[1]: Stopping Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:49:04.370 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[23696]: Upgrade: Updating mon.a 2024-09-17T01:49:04.370 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[23696]: Deploying daemon mon.a on smithi081 2024-09-17T01:49:04.370 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[23696]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.370 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[23696]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.371 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[28146]: Upgrade: Updating mon.a 2024-09-17T01:49:04.371 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[28146]: Deploying daemon mon.a on smithi081 2024-09-17T01:49:04.371 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[28146]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.371 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-mon[28146]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:04 smithi105 ceph-mon[63427]: Upgrade: Updating mon.a 2024-09-17T01:49:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:04 smithi105 ceph-mon[63427]: Deploying daemon mon.a on smithi081 2024-09-17T01:49:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:04 smithi105 ceph-mon[63427]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:04 smithi105 ceph-mon[63427]: from='client.25522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:04.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[23692]: 2024-09-17T01:49:04.371+0000 7f68e265e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:49:04.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[23692]: 2024-09-17T01:49:04.371+0000 7f68e265e700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T01:49:04.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 podman[77693]: 2024-09-17 01:49:04.402385501 +0000 UTC m=+0.147291926 container died b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.tags=base centos centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T01:49:04.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 podman[77693]: 2024-09-17 01:49:04.788205768 +0000 UTC m=+0.533112187 container cleanup b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, vendor=Red Hat, Inc., version=8, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T01:49:04.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:04 smithi081 bash[77693]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a 2024-09-17T01:49:05.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 podman[77708]: 2024-09-17 01:49:05.064645211 +0000 UTC m=+0.663330094 container remove b8a531e37111fb3a471692ed5df66230aadf13269390876ae68f26c42d4144b3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, build-date=2022-05-03T08:36:31.336870, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-09-17T01:49:05.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-17T01:49:05.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 systemd[1]: Stopped Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:49:05.393 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service: Consumed 24.168s CPU time. 2024-09-17T01:49:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 systemd[1]: Starting Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:49:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 podman[77836]: 2024-09-17 01:49:05.505198414 +0000 UTC m=+0.015567698 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:49:05.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 podman[77836]: 2024-09-17 01:49:05.607305511 +0000 UTC m=+0.117674794 container create 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T01:49:06.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 podman[77836]: 2024-09-17 01:49:05.818153561 +0000 UTC m=+0.328522850 container init 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 podman[77836]: 2024-09-17 01:49:05.821413527 +0000 UTC m=+0.331782820 container start 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: pidfile_write: ignore empty --pid-file 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: load: jerasure load: lrc 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: RocksDB version: 7.9.2 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Git sha 0 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: DB SUMMARY 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: DB Session ID: OPFGERHD51GQ6YPGVHMX 2024-09-17T01:49:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2904 Bytes 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 1372687 ; 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.env: 0x55c98fd16940 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.info_log: 0x55c991d636e0 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:49:06.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.statistics: (nil) 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.use_fsync: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.db_log_dir: 2024-09-17T01:49:06.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.wal_dir: 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.write_buffer_manager: 0x55c991d728c0 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:49:06.184 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.unordered_write: 0 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.row_cache: None 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.wal_filter: None 2024-09-17T01:49:06.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:49:06.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:49:06.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:49:06.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.wal_compression: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T01:49:06.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_open_files: -1 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:49:06.188 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Compression algorithms supported: 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kZSTD supported: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:49:06.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.merge_operator: 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_filter: None 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c991d63320) 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:49:06.190 INFO:journalctl@ceph.mon.a.smithi081.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_type: 0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: data_block_index_type: 0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_shortening: 1 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: checksum: 4 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: no_block_cache: 0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache: 0x55c991d571f0 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_options: 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: capacity : 536870912 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: num_shard_bits : 4 2024-09-17T01:49:06.191 INFO:journalctl@ceph.mon.a.smithi081.stdout: strict_capacity_limit : 0 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_cache_compressed: (nil) 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: persistent_cache: (nil) 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_size: 4096 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_size_deviation: 10 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_restart_interval: 16 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: index_block_restart_interval: 1 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: metadata_block_size: 4096 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: partition_filters: 0 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: use_delta_encoding: 1 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: filter_policy: bloomfilter 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: whole_key_filtering: 1 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: verify_compression: 0 2024-09-17T01:49:06.192 INFO:journalctl@ceph.mon.a.smithi081.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: format_version: 5 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: enable_index_compression: 1 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: block_align: 0 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: max_auto_readahead_size: 262144 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: prepopulate_block_cache: 0 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: initial_auto_readahead_size: 8192 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression: NoCompression 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:49:06.193 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.num_levels: 7 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:49:06.194 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:49:06.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:49:06.196 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:49:06.197 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.ttl: 2592000 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T01:49:06.198 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26753, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-09-17T01:49:06.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f70ea838-950b-444a-859d-c061f83b90cc 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537745848093, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537745853338, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1289246, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26754, "largest_seqno": 28184, "table_properties": {"data_size": 1283578, "index_size": 3027, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 15977, "raw_average_key_size": 25, "raw_value_size": 1270718, "raw_average_value_size": 1994, "num_data_blocks": 136, "num_entries": 637, "num_filter_entries": 637, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726537745, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f70ea838-950b-444a-859d-c061f83b90cc", "db_session_id": "OPFGERHD51GQ6YPGVHMX", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537745853443, "job": 1, "event": "recovery_finished"} 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c991d84e00 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: DB pointer 0x55c991e80000 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** DB Stats ** 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:49:06.200 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: L0 1/0 1.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 439.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: L6 1/0 8.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Sum 2/0 9.62 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 439.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:06.201 INFO:journalctl@ceph.mon.a.smithi081.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 439.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 439.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:49:06.202 INFO:journalctl@ceph.mon.a.smithi081.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: Cumulative compaction: 0.00 GB write, 126.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: Interval compaction: 0.00 GB write, 126.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: Block cache BinnedLRUCache@0x55c991d571f0#2 capacity: 512.00 MB usage: 4.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: starting mon.a rank 0 at public addrs [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] at bind addrs [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???) e3 preinit fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).mds e1 new map 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).mds e1 print_map 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: e1 2024-09-17T01:49:06.203 INFO:journalctl@ceph.mon.a.smithi081.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: legacy client fscid: -1 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout: No filesystems configured 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 ceph-mon[77854]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 bash[77836]: 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 2024-09-17T01:49:06.204 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:05 smithi081 systemd[1]: Started Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:49:09.257 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: ignoring --setuser ceph since I am not root 2024-09-17T01:49:09.257 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: ignoring --setgroup ceph since I am not root 2024-09-17T01:49:09.257 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mgr[59592]: -- 172.21.15.105:0/3838913036 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f21baa70e0 con 0x55f21ba85000 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.055+0000 7f916a2c5040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.112+0000 7f916a2c5040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.259+0000 7f916a2c5040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: mon.a calling monitor election 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: monmap epoch 3 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: last_changed 2024-09-17T01:21:24.676494+0000 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:09.258 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: min_mon_release 17 (quincy) 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: election_strategy: 1 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: fsmap 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: from='mgr.15267 ' entity='' 2024-09-17T01:49:09.259 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:08 smithi105 ceph-mon[63427]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-17T01:49:09.268 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ignoring --setuser ceph since I am not root 2024-09-17T01:49:09.268 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ignoring --setgroup ceph since I am not root 2024-09-17T01:49:09.268 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mgr[60921]: -- 172.21.15.81:0/2155041835 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5649a94290e0 con 0x5649a9407000 2024-09-17T01:49:09.268 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.056+0000 7f46fd842040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.113+0000 7f46fd842040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.270+0000 7f46fd842040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: mon.a calling monitor election 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: monmap epoch 3 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: last_changed 2024-09-17T01:21:24.676494+0000 2024-09-17T01:49:09.269 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: min_mon_release 17 (quincy) 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: election_strategy: 1 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: fsmap 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: from='mgr.15267 ' entity='' 2024-09-17T01:49:09.270 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[28146]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: mon.a calling monitor election 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: monmap epoch 3 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: last_changed 2024-09-17T01:21:24.676494+0000 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: min_mon_release 17 (quincy) 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: election_strategy: 1 2024-09-17T01:49:09.271 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: fsmap 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: mgrmap e43: y(active, since 2m), standbys: x 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in unknown state 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: from='mgr.15267 ' entity='' 2024-09-17T01:49:09.272 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:08 smithi081 ceph-mon[77854]: mgrmap e44: y(active, since 2m), standbys: x 2024-09-17T01:49:09.607 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.324+0000 7f916a2c5040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:49:09.608 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.468+0000 7f916a2c5040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:49:09.621 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.337+0000 7f46fd842040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T01:49:09.621 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.483+0000 7f46fd842040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:49:09.621 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.623+0000 7f46fd842040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:49:09.885 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.609+0000 7f916a2c5040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T01:49:10.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.928+0000 7f916a2c5040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:49:10.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:09.988+0000 7f916a2c5040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:49:10.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:10 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:10.172+0000 7f916a2c5040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:49:10.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.938+0000 7f46fd842040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T01:49:10.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:10 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:09.999+0000 7f46fd842040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T01:49:10.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:10 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:10.185+0000 7f46fd842040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T01:49:11.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:10 smithi081 ceph-mon[77854]: mgrmap e45: y(active, since 2m), standbys: x 2024-09-17T01:49:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:10 smithi081 ceph-mon[28146]: mgrmap e45: y(active, since 2m), standbys: x 2024-09-17T01:49:11.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:11.076+0000 7f46fd842040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:49:11.181 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:11.143+0000 7f46fd842040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:49:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:10 smithi105 ceph-mon[63427]: mgrmap e45: y(active, since 2m), standbys: x 2024-09-17T01:49:11.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:11.065+0000 7f916a2c5040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T01:49:11.385 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:11.132+0000 7f916a2c5040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T01:49:12.095 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:11.840+0000 7f916a2c5040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:49:12.095 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:11 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:11.956+0000 7f916a2c5040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:49:12.108 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:11.851+0000 7f46fd842040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T01:49:12.108 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:11.967+0000 7f46fd842040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T01:49:12.374 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.097+0000 7f916a2c5040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:49:12.375 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.159+0000 7f916a2c5040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:49:12.375 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.309+0000 7f916a2c5040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:49:12.375 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.376+0000 7f916a2c5040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:49:12.388 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.110+0000 7f46fd842040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T01:49:12.388 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.171+0000 7f46fd842040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T01:49:12.388 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.319+0000 7f46fd842040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T01:49:12.388 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.389+0000 7f46fd842040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T01:49:12.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.436+0000 7f916a2c5040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:49:12.635 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.594+0000 7f916a2c5040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:49:12.672 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.450+0000 7f46fd842040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T01:49:12.672 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.607+0000 7f46fd842040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T01:49:12.928 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.660+0000 7f916a2c5040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:49:12.928 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.729+0000 7f916a2c5040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:49:12.928 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.811+0000 7f916a2c5040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:49:12.928 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.871+0000 7f916a2c5040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:49:12.928 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:12.929+0000 7f916a2c5040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:49:12.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.674+0000 7f46fd842040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T01:49:12.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.740+0000 7f46fd842040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T01:49:12.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.823+0000 7f46fd842040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T01:49:12.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.883+0000 7f46fd842040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T01:49:13.430 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:12.942+0000 7f46fd842040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:13.718+0000 7f916a2c5040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:49:13] ENGINE Bus STARTING 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: CherryPy Checker: 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: The Application mounted at '' has an empty config. 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:49:13] ENGINE Serving on http://:::9283 2024-09-17T01:49:13.972 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x[59588]: [17/Sep/2024:01:49:13] ENGINE Bus STARTED 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: Standby manager daemon x restarted 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: Standby manager daemon x started 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: Active manager daemon y restarted 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: Activating manager daemon y 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:13.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: mgrmap e46: y(active, starting, since 0.00891318s), standbys: x 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:49:13.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:49:13.975 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:49:13.975 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:49:13.975 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:49:13.975 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:13 smithi105 ceph-mon[63427]: Manager daemon y is now available 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: Standby manager daemon x restarted 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: Standby manager daemon x started 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: Active manager daemon y restarted 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: Activating manager daemon y 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:13.986 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: mgrmap e46: y(active, starting, since 0.00891318s), standbys: x 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:49:13.987 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[77854]: Manager daemon y is now available 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:13.730+0000 7f46fd842040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T01:49:13.988 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:49:13] ENGINE Bus STARTING 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: CherryPy Checker: 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: The Application mounted at '' has an empty config. 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:49:13] ENGINE Serving on http://:::9283 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:49:13] ENGINE Bus STARTED 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: Standby manager daemon x restarted 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: Standby manager daemon x started 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.? 172.21.15.105:0/287407728' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:49:13.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: Active manager daemon y restarted 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: Activating manager daemon y 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: mgrmap e46: y(active, starting, since 0.00891318s), standbys: x 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:49:13.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:49:13.991 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:49:13.991 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:49:13.991 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:49:13.991 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:49:13.991 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:13 smithi081 ceph-mon[28146]: Manager daemon y is now available 2024-09-17T01:49:14.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:49:14.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:49:15.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:15.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:49:15.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:49:15.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:14 smithi105 ceph-mon[63427]: mgrmap e47: y(active, since 1.01105s), standbys: x 2024-09-17T01:49:15.044 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:15.044 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:49:15.044 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:49:15.044 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[77854]: mgrmap e47: y(active, since 1.01105s), standbys: x 2024-09-17T01:49:15.045 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:14.743+0000 7f469c54b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-17T01:49:15.045 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:15.045 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:49:15.045 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T01:49:15.045 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-mon[28146]: mgrmap e47: y(active, since 1.01105s), standbys: x 2024-09-17T01:49:15.045 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:49:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:49:14.766Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=2 2024-09-17T01:49:15.786 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:15.477+0000 7f46545f4640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:01:49:15] ENGINE Error in HTTPServer.serve 2024-09-17T01:49:15.786 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: Traceback (most recent call last): 2024-09-17T01:49:15.786 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._connections.run(self.expiration_interval) 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._run(expiration_interval) 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: s = self.context.wrap_socket( 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: return self.sslsocket_class._create( 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:49:15.787 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self.do_handshake() 2024-09-17T01:49:15.788 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:49:15.788 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: self._sslobj.do_handshake() 2024-09-17T01:49:15.788 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:49:15.788 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:15.858 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:15.858 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:15 smithi105 ceph-mon[63427]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:15.858 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:15 smithi105 ceph-mon[63427]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T01:49:15.858 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:15 smithi105 ceph-mon[63427]: Cluster is now healthy 2024-09-17T01:49:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[77854]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[77854]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T01:49:16.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[77854]: Cluster is now healthy 2024-09-17T01:49:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[28146]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[28146]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T01:49:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:15 smithi081 ceph-mon[28146]: Cluster is now healthy 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: [17/Sep/2024:01:49:15] ENGINE Bus STARTING 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: [17/Sep/2024:01:49:15] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: [17/Sep/2024:01:49:15] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: [17/Sep/2024:01:49:15] ENGINE Bus STARTED 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: [17/Sep/2024:01:49:15] ENGINE Error in HTTPServer.serve 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:49:17.104 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: self.do_handshake() 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:17.105 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[77854]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: [17/Sep/2024:01:49:15] ENGINE Bus STARTING 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: [17/Sep/2024:01:49:15] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: [17/Sep/2024:01:49:15] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: [17/Sep/2024:01:49:15] ENGINE Bus STARTED 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: [17/Sep/2024:01:49:15] ENGINE Error in HTTPServer.serve 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: Traceback (most recent call last): 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._run(expiration_interval) 2024-09-17T01:49:17.106 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: s = self.context.wrap_socket( 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: return self.sslsocket_class._create( 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: self.do_handshake() 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: self._sslobj.do_handshake() 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:17.107 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:16 smithi081 ceph-mon[28146]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-17T01:49:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: [17/Sep/2024:01:49:15] ENGINE Bus STARTING 2024-09-17T01:49:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: [17/Sep/2024:01:49:15] ENGINE Serving on http://172.21.15.81:8765 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: [17/Sep/2024:01:49:15] ENGINE Serving on https://172.21.15.81:7150 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: [17/Sep/2024:01:49:15] ENGINE Bus STARTED 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: [17/Sep/2024:01:49:15] ENGINE Error in HTTPServer.serve 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:49:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:17.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:16 smithi105 ceph-mon[63427]: mgrmap e48: y(active, since 2s), standbys: x 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:18 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: mgrmap e49: y(active, since 4s), standbys: x 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:18 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[28146]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:20.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:20 smithi081 ceph-mon[77854]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:20.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:20.590 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:20 smithi105 ceph-mon[63427]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:49:23.134 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[77854]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:49:23.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:23.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:23.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[28146]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:49:23.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:23.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:22 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:23.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:22 smithi105 ceph-mon[63427]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T01:49:23.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:23.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:23.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:23.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:49:23] "GET /metrics HTTP/1.1" 200 35209 "" "Prometheus/2.43.0" 2024-09-17T01:49:24.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:49:24.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:24.060 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.061 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:24.062 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:24.062 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:24.062 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:23 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T01:49:24.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi081:/etc/ceph/ceph.conf 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: Updating smithi081:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:49:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T01:49:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: Upgrade: It appears safe to stop mon.c 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: Upgrade: It appears safe to stop mon.c 2024-09-17T01:49:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:49:24.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:24.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:24.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:24.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:24 smithi081 ceph-mon[28146]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:25.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: Updating smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:25.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: Updating smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:49:25.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T01:49:25.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: Upgrade: It appears safe to stop mon.c 2024-09-17T01:49:25.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T01:49:25.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:25.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:25.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:25.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:26.053 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[77854]: Upgrade: Updating mon.c 2024-09-17T01:49:26.053 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[77854]: Deploying daemon mon.c on smithi081 2024-09-17T01:49:26.053 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:26.054 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[28146]: Upgrade: Updating mon.c 2024-09-17T01:49:26.054 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[28146]: Deploying daemon mon.c on smithi081 2024-09-17T01:49:26.054 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:25 smithi081 ceph-mon[28146]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:25 smithi105 ceph-mon[63427]: Upgrade: Updating mon.c 2024-09-17T01:49:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:25 smithi105 ceph-mon[63427]: Deploying daemon mon.c on smithi081 2024-09-17T01:49:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:26.597 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 systemd[1]: Stopping Ceph mon.c for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:49:26.597 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c[28142]: 2024-09-17T01:49:26.568+0000 7f2540439700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:49:26.597 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c[28142]: 2024-09-17T01:49:26.568+0000 7f2540439700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T01:49:26.859 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 podman[80026]: 2024-09-17 01:49:26.599455459 +0000 UTC m=+0.125652928 container died 4f17337c12f699a17dfe601c779e14939651c54eb29c248eef265c69ffc6a4d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.19.8, version=8, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, name=centos-stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T01:49:27.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 podman[80026]: 2024-09-17 01:49:26.951675983 +0000 UTC m=+0.477873450 container cleanup 4f17337c12f699a17dfe601c779e14939651c54eb29c248eef265c69ffc6a4d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., release=754, io.openshift.tags=base centos centos-stream, version=8, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-type=git) 2024-09-17T01:49:27.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:26 smithi081 bash[80026]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c 2024-09-17T01:49:27.570 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 podman[80038]: 2024-09-17 01:49:27.236261387 +0000 UTC m=+0.629532722 container remove 4f17337c12f699a17dfe601c779e14939651c54eb29c248eef265c69ffc6a4d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T01:49:27.570 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-17T01:49:27.571 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 systemd[1]: Stopped Ceph mon.c for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:49:27.571 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.c.service: Consumed 14.049s CPU time. 2024-09-17T01:49:27.571 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 systemd[1]: Starting Ceph mon.c for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:49:27.925 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 podman[80164]: 2024-09-17 01:49:27.683226708 +0000 UTC m=+0.015955604 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:27 smithi081 podman[80164]: 2024-09-17 01:49:27.928025667 +0000 UTC m=+0.260754553 container create c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 podman[80164]: 2024-09-17 01:49:28.148123301 +0000 UTC m=+0.480852197 container init c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 podman[80164]: 2024-09-17 01:49:28.151368532 +0000 UTC m=+0.484097420 container start c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: pidfile_write: ignore empty --pid-file 2024-09-17T01:49:28.176 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: load: jerasure load: lrc 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: RocksDB version: 7.9.2 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Git sha 0 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: DB SUMMARY 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: DB Session ID: EKAVP0ZY6TIIK5PKCXJI 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3019 Bytes 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000057.sst 2024-09-17T01:49:28.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000055.log size: 2325128 ; 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.env: 0x55a21c9c8940 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.info_log: 0x55a21e0c16e0 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.statistics: (nil) 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.use_fsync: 0 2024-09-17T01:49:28.178 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.db_log_dir: 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.wal_dir: 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:49:28.179 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.write_buffer_manager: 0x55a21e0d08c0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:49:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.unordered_write: 0 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:49:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.row_cache: None 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.wal_filter: None 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.wal_compression: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:49:28.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:49:28.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_open_files: -1 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Compression algorithms supported: 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kZSTD supported: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:49:28.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:49:28.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:49:28.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:49:28.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[77854]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:28.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.merge_operator: 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_filter: None 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:49:28.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a21e0c1320) 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: index_type: 0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: data_block_index_type: 0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: index_shortening: 1 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: checksum: 4 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: no_block_cache: 0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_cache: 0x55a21e0b51f0 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_cache_options: 2024-09-17T01:49:28.432 INFO:journalctl@ceph.mon.c.smithi081.stdout: capacity : 536870912 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: num_shard_bits : 4 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: strict_capacity_limit : 0 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_cache_compressed: (nil) 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: persistent_cache: (nil) 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_size: 4096 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_size_deviation: 10 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_restart_interval: 16 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: index_block_restart_interval: 1 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: metadata_block_size: 4096 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: partition_filters: 0 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: use_delta_encoding: 1 2024-09-17T01:49:28.433 INFO:journalctl@ceph.mon.c.smithi081.stdout: filter_policy: bloomfilter 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: whole_key_filtering: 1 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: verify_compression: 0 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: format_version: 5 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: enable_index_compression: 1 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: block_align: 0 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: max_auto_readahead_size: 262144 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: prepopulate_block_cache: 0 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: initial_auto_readahead_size: 8192 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:49:28.434 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression: NoCompression 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.num_levels: 7 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:49:28.435 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:49:28.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:49:28.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:49:28.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:49:28.439 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:49:28.440 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.ttl: 2592000 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T01:49:28.441 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 57.sst 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 59, last_sequence is 30887, log_number is 55,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 55 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 68242ab1-31f7-47cc-b490-ed82641916ca 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537768178398, "job": 1, "event": "recovery_started", "wal_files": [55]} 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #55 mode 2 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537768184988, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 60, "file_size": 1482222, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 30892, "largest_seqno": 31285, "table_properties": {"data_size": 1479320, "index_size": 1350, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4637, "raw_average_key_size": 24, "raw_value_size": 1474966, "raw_average_value_size": 7722, "num_data_blocks": 60, "num_entries": 191, "num_filter_entries": 191, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726537768, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "68242ab1-31f7-47cc-b490-ed82641916ca", "db_session_id": "EKAVP0ZY6TIIK5PKCXJI", "orig_file_number": 60, "seqno_to_time_mapping": "N/A"}} 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726537768185145, "job": 1, "event": "recovery_finished"} 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:5047] Creating manifest 62 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000055.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:49:28.442 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a21e0e2e00 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: DB pointer 0x55a21e1de000 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: ** DB Stats ** 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:49:28.443 INFO:journalctl@ceph.mon.c.smithi081.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: L0 1/0 1.41 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: L6 1/0 10.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: Sum 2/0 11.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: ** Compaction Stats [default] ** 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 517.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T01:49:28.444 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Cumulative compaction: 0.00 GB write, 123.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Interval compaction: 0.00 GB write, 123.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Block cache BinnedLRUCache@0x55a21e0b51f0#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-09-17T01:49:28.445 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: starting mon.c rank 1 at public addrs [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] at bind addrs [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???) e3 preinit fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).mds e1 new map 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).mds e1 print_map 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: e1 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: legacy client fscid: -1 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout: No filesystems configured 2024-09-17T01:49:28.446 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 ceph-mon[80183]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 bash[80164]: c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da 2024-09-17T01:49:28.447 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:28 smithi081 systemd[1]: Started Ceph mon.c for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:49:28.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:28 smithi105 ceph-mon[63427]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mon.a calling monitor election 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mon.c calling monitor election 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mon.b calling monitor election 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: monmap epoch 4 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: last_changed 2024-09-17T01:49:28.225558+0000 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:29.638 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: min_mon_release 19 (squid) 2024-09-17T01:49:29.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: election_strategy: 1 2024-09-17T01:49:29.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:29.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:29.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:29.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: fsmap 2024-09-17T01:49:29.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:29.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:29.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.670 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mon.a calling monitor election 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mon.c calling monitor election 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mon.b calling monitor election 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:29.671 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: monmap epoch 4 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: last_changed 2024-09-17T01:49:28.225558+0000 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:29.672 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: min_mon_release 19 (squid) 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: election_strategy: 1 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: fsmap 2024-09-17T01:49:29.673 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.674 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.675 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:29.675 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.675 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.675 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mon.a calling monitor election 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mon.c calling monitor election 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mon.b calling monitor election 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T01:49:29.676 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T01:49:29.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T01:49:29.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T01:49:29.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: monmap epoch 4 2024-09-17T01:49:29.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: fsid ded6f264-7492-11ef-bceb-c7b262605968 2024-09-17T01:49:29.677 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: last_changed 2024-09-17T01:49:28.225558+0000 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: created 2024-09-17T01:19:43.267953+0000 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: min_mon_release 19 (squid) 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: election_strategy: 1 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: 0: [v2:172.21.15.81:3300/0,v1:172.21.15.81:6789/0] mon.a 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: 1: [v2:172.21.15.81:3301/0,v1:172.21.15.81:6790/0] mon.c 2024-09-17T01:49:29.678 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: 2: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-17T01:49:29.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: fsmap 2024-09-17T01:49:29.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:49:29.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: mgrmap e49: y(active, since 14s), standbys: x 2024-09-17T01:49:29.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:49:29.679 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:49:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:49:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:29.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:30.513 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:30 smithi081 ceph-mon[77854]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:49:30.513 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:30 smithi081 ceph-mon[80183]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:49:30.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:30 smithi105 ceph-mon[63427]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T01:49:32.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:32 smithi105 ceph-mon[63427]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:32.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:32 smithi081 ceph-mon[77854]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:32.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:32 smithi081 ceph-mon[80183]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T01:49:33.481 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:49:33.197+0000 7f469c54b640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-17T01:49:33.660 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:49:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:49:33] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.43.0" 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[77854]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.354 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-17T01:49:34.355 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.355 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.355 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:34 smithi081 ceph-mon[80183]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (15m) 1s ago 23m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (2m) 15s ago 23m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 1s ago 22m 58.4M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (2m) 15s ago 27m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:49:34.362 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (11m) 1s ago 29m 488M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (28s) 1s ago 29m 40.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (77s) 15s ago 28m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (6s) 1s ago 28m 19.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (15m) 1s ago 24m 15.1M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (15m) 15s ago 24m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (27m) 1s ago 27m 55.1M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (27m) 1s ago 27m 56.2M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (26m) 1s ago 26m 51.3M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (26m) 1s ago 26m 54.4M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (26m) 15s ago 26m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (25m) 15s ago 25m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:49:34.363 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (25m) 15s ago 25m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:49:34.364 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (24m) 15s ago 24m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:49:34.364 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (2m) 15s ago 24m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:49:34.364 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (22m) 1s ago 22m 78.9M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:49:34.364 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (22m) 15s ago 22m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:34 smithi105 ceph-mon[63427]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:49:34.762 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:49:34.763 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:49:35.103 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi081", 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [ 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "mon" 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:49:35.104 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:49:35.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[77854]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[77854]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2821034276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:35.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:35.319 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[80183]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.319 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[80183]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.319 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2821034276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:35.319 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:35 smithi105 ceph-mon[63427]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:35 smithi105 ceph-mon[63427]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:35 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2821034276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:35.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:36 smithi105 ceph-mon[63427]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:36 smithi105 ceph-mon[63427]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:36.416 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:36 smithi081 ceph-mon[77854]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:36.417 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:36 smithi081 ceph-mon[77854]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:36.417 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:36 smithi081 ceph-mon[80183]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:49:36.417 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:36 smithi081 ceph-mon[80183]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: Detected new or changed devices on smithi081 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:37.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: Reconfiguring mon.a (monmap changed)... 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[77854]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: Detected new or changed devices on smithi081 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: Reconfiguring mon.a (monmap changed)... 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:37.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:37 smithi081 ceph-mon[80183]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:49:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: Detected new or changed devices on smithi081 2024-09-17T01:49:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 891 B/s rd, 0 op/s 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: Reconfiguring mon.a (monmap changed)... 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:38.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:37 smithi105 ceph-mon[63427]: Reconfiguring daemon mon.a on smithi081 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:39.531 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[77854]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:49:39.532 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:39 smithi081 ceph-mon[80183]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: Reconfiguring daemon mgr.y on smithi081 2024-09-17T01:49:39.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:39 smithi105 ceph-mon[63427]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:40.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:49:40.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:40.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:40.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:40.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:40 smithi105 ceph-mon[63427]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:49:40.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:49:40.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:40.904 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[77854]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: Reconfiguring mon.c (monmap changed)... 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:40.905 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:40 smithi081 ceph-mon[80183]: Reconfiguring daemon mon.c on smithi081 2024-09-17T01:49:42.245 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:42.245 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.245 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.245 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T01:49:42.245 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.0 on smithi081 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:42.246 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:41 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.0 on smithi081 2024-09-17T01:49:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:42.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:41 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.0 on smithi081 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:49:43] "GET /metrics HTTP/1.1" 200 37963 "" "Prometheus/2.43.0" 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.1 on smithi081 2024-09-17T01:49:43.610 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[77854]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.1 on smithi081 2024-09-17T01:49:43.611 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:43 smithi081 ceph-mon[80183]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T01:49:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:49:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.1 on smithi081 2024-09-17T01:49:43.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:43 smithi105 ceph-mon[63427]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T01:49:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:49:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.2 on smithi081 2024-09-17T01:49:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T01:49:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:49:45.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.2 on smithi081 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.2 on smithi081 2024-09-17T01:49:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:49:45.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:49:45.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:49:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:47.001 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T01:49:47.001 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.3 on smithi081 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.3 on smithi081 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.002 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:47.003 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T01:49:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.3 on smithi081 2024-09-17T01:49:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:48.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:47 smithi105 ceph-mon[63427]: Reconfiguring rgw.foo.smithi081.fnhdrk (monmap changed)... 2024-09-17T01:49:48.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:47 smithi105 ceph-mon[63427]: Reconfiguring daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:49:48.114 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:47 smithi105 ceph-mon[63427]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[77854]: Reconfiguring rgw.foo.smithi081.fnhdrk (monmap changed)... 2024-09-17T01:49:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[77854]: Reconfiguring daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:49:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[77854]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:48.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[80183]: Reconfiguring rgw.foo.smithi081.fnhdrk (monmap changed)... 2024-09-17T01:49:48.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[80183]: Reconfiguring daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:49:48.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:47 smithi081 ceph-mon[80183]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:49.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:48 smithi105 ceph-mon[63427]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:49:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[77854]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: Reconfiguring mon.b (monmap changed)... 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:48 smithi081 ceph-mon[80183]: Reconfiguring daemon mon.b on smithi105 2024-09-17T01:49:50.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:50.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:49:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:50.995 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T01:49:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T01:49:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:49:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T01:49:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T01:49:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:49:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T01:49:51.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T01:49:51.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:51.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:51.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:49:51.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:52.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:51 smithi105 ceph-mon[63427]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T01:49:52.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:51 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T01:49:52.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:51 smithi105 ceph-mon[63427]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[77854]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T01:49:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T01:49:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[77854]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[80183]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T01:49:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T01:49:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:51 smithi081 ceph-mon[80183]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T01:49:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:49:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:52 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:52 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T01:49:53.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:49:53] "GET /metrics HTTP/1.1" 200 38034 "" "Prometheus/2.43.0" 2024-09-17T01:49:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:53 smithi105 ceph-mon[63427]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:49:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[77854]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[80183]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:49:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:55.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T01:49:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T01:49:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:49:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T01:49:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T01:49:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:49:55.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:55.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T01:49:55.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T01:49:55.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:55.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:49:55.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:56.129 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:55 smithi105 ceph-mon[63427]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T01:49:56.129 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:55 smithi105 ceph-mon[63427]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T01:49:56.129 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:55 smithi105 ceph-mon[63427]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:56.129 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:55 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[77854]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T01:49:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[77854]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T01:49:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[77854]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:56.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[80183]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T01:49:56.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[80183]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T01:49:56.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[80183]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:49:56.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:55 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: Reconfiguring rgw.foo.smithi105.ttsowb (monmap changed)... 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: Reconfiguring daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:49:57.486 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:57 smithi105 ceph-mon[63427]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: Reconfiguring rgw.foo.smithi105.ttsowb (monmap changed)... 2024-09-17T01:49:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: Reconfiguring daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[77854]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: Reconfiguring rgw.foo.smithi105.ttsowb (monmap changed)... 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: Reconfiguring daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:49:57.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:57 smithi081 ceph-mon[80183]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:49:58.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T01:49:58.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mds 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nfs 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all grafana 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all loki 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all promtail 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:49:58.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:49:58.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:58.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-17T01:49:58.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:49:58.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:49:58.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mds 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:49:58.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nfs 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mds 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:49:58.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nfs 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.937 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all grafana 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all loki 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all promtail 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Finalizing container_image settings 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all node-exporter 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.939 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all prometheus 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all alertmanager 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all grafana 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all loki 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all promtail 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Finalizing container_image settings 2024-09-17T01:49:58.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:49:58.943 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:49:58.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.945 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.946 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: Upgrade: Complete! 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.947 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: Upgrade: Complete! 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.948 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 937 B/s rd, 0 op/s 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 727 B/s rd, 0 op/s 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:49:58.949 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:00.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:00.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:49:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:00.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:00.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:49:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:49:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:00 smithi105 ceph-mon[63427]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:00 smithi105 ceph-mon[63427]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:50:01.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:00 smithi081 ceph-mon[77854]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:01.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:00 smithi081 ceph-mon[77854]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:50:01.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:00 smithi081 ceph-mon[80183]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:01.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:00 smithi081 ceph-mon[80183]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:50:03.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:02 smithi105 ceph-mon[63427]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:03.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:02 smithi081 ceph-mon[77854]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:03.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:02 smithi081 ceph-mon[80183]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T01:50:03.772 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:03] "GET /metrics HTTP/1.1" 200 38030 "" "Prometheus/2.43.0" 2024-09-17T01:50:05.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:04 smithi105 ceph-mon[63427]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:50:05.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:04 smithi081 ceph-mon[77854]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:50:05.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:04 smithi081 ceph-mon[80183]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:50:06.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:05 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:06.177 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:05 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:06.177 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:05 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:06.202 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:50:07.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:06 smithi105 ceph-mon[63427]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:07.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:06 smithi105 ceph-mon[63427]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:07.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:06 smithi081 ceph-mon[77854]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:07.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:06 smithi081 ceph-mon[77854]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:07.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:06 smithi081 ceph-mon[80183]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:07.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:06 smithi081 ceph-mon[80183]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:08.320 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:50:08.320 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (16m) 35s ago 24m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:50:08.320 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (2m) 49s ago 23m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 35s ago 23m 58.4M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (3m) 49s ago 28m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (12m) 35s ago 30m 488M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (62s) 35s ago 30m 40.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (111s) 49s ago 28m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (40s) 35s ago 28m 19.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (16m) 35s ago 24m 15.1M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (15m) 49s ago 24m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (28m) 35s ago 28m 55.1M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (27m) 35s ago 27m 56.2M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (27m) 35s ago 27m 51.3M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (27m) 35s ago 27m 54.4M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:50:08.321 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (26m) 49s ago 26m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (26m) 49s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (25m) 49s ago 25m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (25m) 49s ago 25m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (3m) 49s ago 24m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (23m) 35s ago 23m 78.9M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:50:08.322 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (23m) 49s ago 23m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:50:08.945 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-09-17T01:50:09.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:08 smithi105 ceph-mon[63427]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:50:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:08 smithi081 ceph-mon[77854]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:50:09.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:08 smithi081 ceph-mon[80183]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:50:10.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:09 smithi081 ceph-mon[80183]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:09 smithi081 ceph-mon[77854]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:09 smithi105 ceph-mon[63427]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:11.014 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:50:11.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:10 smithi081 ceph-mon[77854]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:10 smithi081 ceph-mon[80183]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:10 smithi105 ceph-mon[63427]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:11.653 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-09-17T01:50:12.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:11 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3357057365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:12.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:11 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3357057365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:12.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:11 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3357057365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:13.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:12 smithi081 ceph-mon[77854]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:13.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:12 smithi081 ceph-mon[80183]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:12 smithi105 ceph-mon[63427]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:13.628 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:50:13.903 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:13] "GET /metrics HTTP/1.1" 200 38030 "" "Prometheus/2.43.0" 2024-09-17T01:50:14.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:13 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4180980815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:14.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:14.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:13 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4180980815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:14.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:13 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:14.216 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-09-17T01:50:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:13 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4180980815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:14 smithi081 ceph-mon[77854]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:15.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:14 smithi081 ceph-mon[80183]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:14 smithi105 ceph-mon[63427]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:15 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:16 smithi081 ceph-mon[77854]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:16 smithi081 ceph-mon[80183]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:16 smithi105 ceph-mon[63427]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:17.404 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:50:17.980 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:17 smithi081 ceph-mon[80183]: from='client.54189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:17.980 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:17 smithi081 ceph-mon[77854]: from='client.54189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:18.006 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:50:18.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:17 smithi105 ceph-mon[63427]: from='client.54189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:18 smithi081 ceph-mon[77854]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:18 smithi081 ceph-mon[80183]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:19.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:18 smithi105 ceph-mon[63427]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:19.871 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:50:19.871 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:50:19.871 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:50:19.872 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:50:20.460 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:50:21.022 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:20 smithi081 ceph-mon[77854]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:21.022 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:20 smithi081 ceph-mon[77854]: from='client.54195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:21.023 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:20 smithi081 ceph-mon[80183]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:21.023 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:20 smithi081 ceph-mon[80183]: from='client.54195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:20 smithi105 ceph-mon[63427]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:20 smithi105 ceph-mon[63427]: from='client.54195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:22.527 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:50:22.528 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:50:22.528 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:50:23.141 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:22 smithi081 ceph-mon[77854]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:23.141 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:22 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1226364471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:50:23.141 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:22 smithi081 ceph-mon[80183]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:23.142 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:22 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1226364471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:50:23.164 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-09-17T01:50:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:22 smithi105 ceph-mon[63427]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:22 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1226364471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:50:23.762 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:23] "GET /metrics HTTP/1.1" 200 38028 "" "Prometheus/2.43.0" 2024-09-17T01:50:25.147 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:24 smithi081 ceph-mon[77854]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:25.147 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:24 smithi081 ceph-mon[80183]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:25.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:24 smithi105 ceph-mon[63427]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:50:26.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:26.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:25 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:26.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:26.479 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:27.113 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:27.113 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:27.114 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:27.144 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:50:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:27.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[77854]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[77854]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[77854]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[77854]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[80183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[80183]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[80183]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:28.195 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:27 smithi081 ceph-mon[80183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:27 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:27 smithi105 ceph-mon[63427]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-09-17T01:50:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:27 smithi105 ceph-mon[63427]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:28.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:27 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mgr 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:50:29.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: Upgrade: osd.0 is safe to restart 2024-09-17T01:50:29.516 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[77854]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mgr 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.517 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: Upgrade: osd.0 is safe to restart 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:29.518 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:29 smithi081 ceph-mon[80183]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-17T01:50:29.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:50:29.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:50:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: Upgrade: osd.0 is safe to restart 2024-09-17T01:50:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:29.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:29 smithi105 ceph-mon[63427]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-09-17T01:50:30.157 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:50:30.819 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:50:30.819 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (16m) 57s ago 24m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:50:30.819 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (3m) 72s ago 24m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:50:30.819 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 57s ago 23m 58.4M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (3m) 72s ago 28m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (12m) 57s ago 30m 488M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (85s) 57s ago 30m 40.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (2m) 72s ago 29m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (63s) 57s ago 29m 19.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (16m) 57s ago 25m 15.1M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (16m) 72s ago 25m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (28m) 57s ago 28m 55.1M 2992M 17.2.0 e1d6a67b021e db3cc848e3cd 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (28m) 57s ago 28m 56.2M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (27m) 57s ago 27m 51.3M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (27m) 57s ago 27m 54.4M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:50:30.820 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (27m) 72s ago 27m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (26m) 72s ago 26m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (26m) 72s ago 26m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (25m) 72s ago 25m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (3m) 72s ago 25m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (23m) 57s ago 23m 78.9M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:50:30.821 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (23m) 72s ago 23m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:50:31.239 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:50:31.240 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:50:31.241 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:50:31.582 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "0/8 daemons upgraded", 2024-09-17T01:50:31.583 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:50:31.583 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:50:31.583 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:50:32.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='client.54225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:32.253 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: Upgrade: Updating osd.0 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: Deploying daemon osd.0 on smithi081 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2034486245' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='client.54225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: Upgrade: Updating osd.0 2024-09-17T01:50:32.254 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:32.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:50:32.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:32.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: Deploying daemon osd.0 on smithi081 2024-09-17T01:50:32.255 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:31 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2034486245' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:32.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='client.54225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: Upgrade: Updating osd.0 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: Deploying daemon osd.0 on smithi081 2024-09-17T01:50:32.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:31 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2034486245' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:32 smithi081 ceph-mon[77854]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:33.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:32 smithi081 ceph-mon[80183]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:32 smithi105 ceph-mon[63427]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:50:33.846 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:33 smithi081 systemd[1]: Stopping Ceph osd.0 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:50:33.846 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:33] "GET /metrics HTTP/1.1" 200 38028 "" "Prometheus/2.43.0" 2024-09-17T01:50:34.106 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:33 smithi081 ceph-mon[77854]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:34.106 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:33 smithi081 ceph-mon[80183]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:33 smithi105 ceph-mon[63427]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:34.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[30992]: 2024-09-17T01:50:34.107+0000 7f14a3e77700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:50:34.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[30992]: 2024-09-17T01:50:34.107+0000 7f14a3e77700 -1 osd.0 113 *** Got signal Terminated *** 2024-09-17T01:50:34.431 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[30992]: 2024-09-17T01:50:34.107+0000 7f14a3e77700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:50:35.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:34 smithi081 ceph-mon[80183]: osd.0 marked itself down and dead 2024-09-17T01:50:35.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:34 smithi081 ceph-mon[77854]: osd.0 marked itself down and dead 2024-09-17T01:50:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:34 smithi105 ceph-mon[63427]: osd.0 marked itself down and dead 2024-09-17T01:50:35.680 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:35 smithi081 podman[86614]: 2024-09-17 01:50:35.445554714 +0000 UTC m=+1.451335855 container died db3cc848e3cd573a81844b2b7ade05a8d973309d44889a675fddc408a4bacd85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux ) 2024-09-17T01:50:35.951 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:35 smithi081 podman[86614]: 2024-09-17 01:50:35.791175805 +0000 UTC m=+1.796956934 container cleanup db3cc848e3cd573a81844b2b7ade05a8d973309d44889a675fddc408a4bacd85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.19.8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, distribution-scope=public, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, version=8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , ceph=True, vendor=Red Hat, Inc.) 2024-09-17T01:50:35.951 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:35 smithi081 bash[86614]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0 2024-09-17T01:50:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:35 smithi105 ceph-mon[63427]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:35 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:50:36.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:35 smithi105 ceph-mon[63427]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T01:50:36.408 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86629]: 2024-09-17 01:50:36.093119071 +0000 UTC m=+0.646406767 container remove db3cc848e3cd573a81844b2b7ade05a8d973309d44889a675fddc408a4bacd85 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, vcs-type=git, name=centos-stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., ceph=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, architecture=x86_64) 2024-09-17T01:50:36.408 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[77854]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[77854]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[80183]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:50:36.409 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:35 smithi081 ceph-mon[80183]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T01:50:36.680 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.313546278 +0000 UTC m=+0.015739684 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:36.680 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.427340527 +0000 UTC m=+0.129533961 container create 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T01:50:36.680 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.605093286 +0000 UTC m=+0.307286691 container init 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:50:36.681 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.60825356 +0000 UTC m=+0.310446963 container start 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:50:36.966 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.713368531 +0000 UTC m=+0.415561924 container attach 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T01:50:36.967 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:36 smithi081 podman[86716]: 2024-09-17 01:50:36.799778061 +0000 UTC m=+0.501971466 container died 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T01:50:37.381 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:36 smithi081 ceph-mon[77854]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T01:50:37.381 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:36 smithi081 ceph-mon[80183]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T01:50:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:36 smithi105 ceph-mon[63427]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T01:50:37.680 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 podman[86738]: 2024-09-17 01:50:37.383764328 +0000 UTC m=+0.574913750 container remove 35cf1d10bc6f15a9b7589aa6dd14cc56cdd52835aadce7df9c92d7333d0267ae (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T01:50:37.681 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T01:50:37.681 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 systemd[1]: Stopped Ceph osd.0 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:50:37.681 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.0.service: Consumed 13.319s CPU time. 2024-09-17T01:50:37.681 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 systemd[1]: Starting Ceph osd.0 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:50:38.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:37 smithi081 ceph-mon[77854]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:38.110 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:37 smithi081 ceph-mon[80183]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:38.110 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 podman[86838]: 2024-09-17 01:50:37.710643501 +0000 UTC m=+0.015620473 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:38.110 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:37 smithi081 podman[86838]: 2024-09-17 01:50:37.826818559 +0000 UTC m=+0.131795525 container create 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS) 2024-09-17T01:50:38.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:37 smithi105 ceph-mon[63427]: pgmap v53: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:38.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 podman[86838]: 2024-09-17 01:50:38.112579882 +0000 UTC m=+0.417556850 container init 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97) 2024-09-17T01:50:38.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 podman[86838]: 2024-09-17 01:50:38.11630267 +0000 UTC m=+0.421279634 container start 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T01:50:38.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 podman[86838]: 2024-09-17 01:50:38.195977672 +0000 UTC m=+0.500954637 container attach 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:50:38.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:50:38.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 bash[86838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:50:38.431 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:50:38.431 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:38 smithi081 bash[86838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:50:39.912 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:50:39.913 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:50:39.913 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:50:39.913 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:50:40.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-mon[80183]: pgmap v54: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:50:40.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-mon[77854]: pgmap v54: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate[86853]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T01:50:40.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 bash[86838]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T01:50:40.182 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:39 smithi081 podman[86838]: 2024-09-17 01:50:39.944304605 +0000 UTC m=+2.249281579 container died 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T01:50:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:39 smithi105 ceph-mon[63427]: pgmap v54: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:50:40.911 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:40 smithi081 podman[87010]: 2024-09-17 01:50:40.574512422 +0000 UTC m=+0.623568829 container remove 6899fd38280d9781535172cc24990c34a949046a0acfe52c0ac13d75ffbc0075 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T01:50:40.911 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:40 smithi081 podman[87073]: 2024-09-17 01:50:40.684974858 +0000 UTC m=+0.016335532 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:50:40.911 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:40 smithi081 podman[87073]: 2024-09-17 01:50:40.800341172 +0000 UTC m=+0.131701819 container create 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:50:41.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:41 smithi081 ceph-mon[77854]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:41.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:41 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 72/723 objects degraded (9.959%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:41.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:41 smithi081 ceph-mon[80183]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:41.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:41 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 72/723 objects degraded (9.959%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:41.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:41 smithi081 podman[87073]: 2024-09-17 01:50:41.019603602 +0000 UTC m=+0.350964263 container init 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T01:50:41.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:41 smithi081 podman[87073]: 2024-09-17 01:50:41.022819909 +0000 UTC m=+0.354180559 container start 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:50:41.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:41 smithi081 ceph-osd[87092]: -- 172.21.15.81:0/2439620205 <== mon.1 v2:172.21.15.81:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c60d75da40 con 0x55c60e524000 2024-09-17T01:50:41.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:41 smithi081 bash[87073]: 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 2024-09-17T01:50:41.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:41 smithi081 systemd[1]: Started Ceph osd.0 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:50:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:41 smithi105 ceph-mon[63427]: pgmap v55: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:41 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 72/723 objects degraded (9.959%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:42.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.431 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:42 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:50:42.402+0000 7f9592a9c640 -1 Falling back to public interface 2024-09-17T01:50:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:42.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:43 smithi081 ceph-mon[77854]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:43 smithi081 ceph-mon[80183]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:43.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:43 smithi105 ceph-mon[63427]: pgmap v56: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:43] "GET /metrics HTTP/1.1" 200 38028 "" "Prometheus/2.43.0" 2024-09-17T01:50:44.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:44.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:44.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:44.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:45 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:45 smithi105 ceph-mon[63427]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 414 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:45 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[80183]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 414 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[77854]: pgmap v57: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 414 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:46.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:50:45.892+0000 7f9592a9c640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T01:50:46.181 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:45 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:50:45.914+0000 7f9592a9c640 -1 osd.0 113 log_to_monitors true 2024-09-17T01:50:47.100 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[77854]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:50:47.100 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.101 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.101 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:50:46.853+0000 7f958a847640 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:50:47.101 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[80183]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:50:47.101 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.101 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:46 smithi105 ceph-mon[63427]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:50:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:47.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[77854]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:50:47.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[77854]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T01:50:47.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[77854]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:50:47.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[77854]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:47 smithi105 ceph-mon[63427]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:50:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:47 smithi105 ceph-mon[63427]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T01:50:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:47 smithi105 ceph-mon[63427]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:50:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:47 smithi105 ceph-mon[63427]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:48.164 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[80183]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:50:48.164 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[80183]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T01:50:48.164 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[80183]: from='osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:50:48.164 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:47 smithi081 ceph-mon[80183]: pgmap v59: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:49.019 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:50:49.019 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[77854]: osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651] boot 2024-09-17T01:50:49.019 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[77854]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T01:50:49.019 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:50:49.019 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:50:49.020 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[80183]: osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651] boot 2024-09-17T01:50:49.020 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[80183]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T01:50:49.020 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:50:49.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:48 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:50:49.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:48 smithi105 ceph-mon[63427]: osd.0 [v2:172.21.15.81:6802/1071392651,v1:172.21.15.81:6803/1071392651] boot 2024-09-17T01:50:49.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:48 smithi105 ceph-mon[63427]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T01:50:49.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:50:49.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:50:49.475+0000 7f469c54b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:50:50.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T01:50:50.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: pgmap v62: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:49 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T01:50:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: pgmap v62: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T01:50:50.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: pgmap v62: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:50:50.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:49 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:50 smithi105 ceph-mon[63427]: pgmap v63: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:50 smithi105 ceph-mon[63427]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:50 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:50 smithi105 ceph-mon[63427]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:50:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[77854]: pgmap v63: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[77854]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[77854]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:50:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[80183]: pgmap v63: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[80183]: pgmap v64: 161 pgs: 40 active+undersized, 18 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 72/723 objects degraded (9.959%) 2024-09-17T01:50:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:50:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:50 smithi081 ceph-mon[80183]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:50:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:51 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:52.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:51 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:51 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:50:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:52 smithi081 ceph-mon[77854]: pgmap v65: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-09-17T01:50:53.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:52 smithi081 ceph-mon[80183]: pgmap v65: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-09-17T01:50:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:52 smithi105 ceph-mon[63427]: pgmap v65: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 32/723 objects degraded (4.426%) 2024-09-17T01:50:53.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:50:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:50:53] "GET /metrics HTTP/1.1" 200 38159 "" "Prometheus/2.43.0" 2024-09-17T01:50:54.267 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:53 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-09-17T01:50:54.268 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:53 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-09-17T01:50:54.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:53 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-09-17T01:50:55.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:54 smithi081 ceph-mon[77854]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-09-17T01:50:55.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:54 smithi081 ceph-mon[80183]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-09-17T01:50:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:54 smithi105 ceph-mon[63427]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-09-17T01:50:56.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:55 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:55 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:55 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:50:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:56 smithi105 ceph-mon[63427]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:50:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:56 smithi081 ceph-mon[77854]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:50:57.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:56 smithi081 ceph-mon[80183]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:50:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:58 smithi105 ceph-mon[63427]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:50:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[77854]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:59.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[80183]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:50:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:50:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:50:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:50:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:00 smithi105 ceph-mon[63427]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:01.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:00 smithi081 ceph-mon[77854]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:00 smithi081 ceph-mon[80183]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:01.949 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (17m) 16s ago 24m 25.0M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (3m) 104s ago 24m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 16s ago 24m 58.5M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (4m) 104s ago 29m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (13m) 16s ago 30m 497M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (117s) 16s ago 31m 45.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:51:02.595 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (2m) 104s ago 29m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (94s) 16s ago 29m 36.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (17m) 16s ago 25m 15.3M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (16m) 104s ago 25m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (21s) 16s ago 29m 12.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (28m) 16s ago 28m 56.0M 2992M 17.2.0 e1d6a67b021e d43b0c25da18 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (28m) 16s ago 28m 51.8M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (27m) 16s ago 27m 55.3M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (27m) 104s ago 27m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (27m) 104s ago 27m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (26m) 104s ago 26m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (26m) 104s ago 26m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:51:02.596 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (4m) 104s ago 25m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:51:02.597 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (24m) 16s ago 24m 79.2M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:51:02.597 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (24m) 104s ago 24m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:51:02.991 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:51:02.992 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:51:03.320 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "1/8 daemons upgraded", 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:51:03.321 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:51:03.321 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:02 smithi081 ceph-mon[77854]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:03.321 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:02 smithi081 ceph-mon[77854]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:03.322 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:02 smithi081 ceph-mon[80183]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:03.322 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:02 smithi081 ceph-mon[80183]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:02 smithi105 ceph-mon[63427]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:02 smithi105 ceph-mon[63427]: from='client.64216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:03.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:03] "GET /metrics HTTP/1.1" 200 38103 "" "Prometheus/2.43.0" 2024-09-17T01:51:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[77854]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[77854]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3308199145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[80183]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[80183]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:03 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3308199145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:03 smithi105 ceph-mon[63427]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:03 smithi105 ceph-mon[63427]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:03 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3308199145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[77854]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[77854]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[80183]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[80183]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:05.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:04 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:04 smithi105 ceph-mon[63427]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:04 smithi105 ceph-mon[63427]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:04 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: Upgrade: osd.1 is safe to restart 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: Upgrade: osd.1 is safe to restart 2024-09-17T01:51:06.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:06.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:06.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:51:06.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:51:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: Upgrade: osd.1 is safe to restart 2024-09-17T01:51:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:06.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:51:06.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[77854]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[77854]: Upgrade: Updating osd.1 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[77854]: Deploying daemon osd.1 on smithi081 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[80183]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[80183]: Upgrade: Updating osd.1 2024-09-17T01:51:07.119 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:06 smithi081 ceph-mon[80183]: Deploying daemon osd.1 on smithi081 2024-09-17T01:51:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:06 smithi105 ceph-mon[63427]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:06 smithi105 ceph-mon[63427]: Upgrade: Updating osd.1 2024-09-17T01:51:07.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:06 smithi105 ceph-mon[63427]: Deploying daemon osd.1 on smithi081 2024-09-17T01:51:08.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:07 smithi081 ceph-mon[77854]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:08.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-mon[80183]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:07 smithi105 ceph-mon[63427]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:08.940 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:08 smithi081 systemd[1]: Stopping Ceph osd.1 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:51:09.327 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-mon[77854]: osd.1 marked itself down and dead 2024-09-17T01:51:09.328 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-mon[80183]: osd.1 marked itself down and dead 2024-09-17T01:51:09.328 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[33929]: 2024-09-17T01:51:08.940+0000 7f85d0436700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:51:09.328 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[33929]: 2024-09-17T01:51:08.940+0000 7f85d0436700 -1 osd.1 118 *** Got signal Terminated *** 2024-09-17T01:51:09.328 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[33929]: 2024-09-17T01:51:08.940+0000 7f85d0436700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:51:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:08 smithi105 ceph-mon[63427]: osd.1 marked itself down and dead 2024-09-17T01:51:09.623 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:09 smithi081 podman[91737]: 2024-09-17 01:51:09.330253761 +0000 UTC m=+0.499495468 container died d43b0c25da18c3ed6cf96ac99cfa134c9cb2820e01e91282a06e8c285c17578c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, io.buildah.version=1.19.8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc.) 2024-09-17T01:51:09.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:09 smithi081 podman[91737]: 2024-09-17 01:51:09.725040037 +0000 UTC m=+0.894281732 container cleanup d43b0c25da18c3ed6cf96ac99cfa134c9cb2820e01e91282a06e8c285c17578c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, architecture=x86_64, release=754, name=centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD) 2024-09-17T01:51:09.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:09 smithi081 bash[91737]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1 2024-09-17T01:51:10.333 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:09 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:51:10.333 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:10 smithi081 ceph-mon[77854]: osdmap e119: 8 total, 7 up, 8 in 2024-09-17T01:51:10.333 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:10 smithi081 ceph-mon[77854]: pgmap v75: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:10.334 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:10 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:51:10.334 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:10 smithi081 ceph-mon[80183]: osdmap e119: 8 total, 7 up, 8 in 2024-09-17T01:51:10.334 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:10 smithi081 ceph-mon[80183]: pgmap v75: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:10.334 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91753]: 2024-09-17 01:51:10.035057079 +0000 UTC m=+0.706946508 container remove d43b0c25da18c3ed6cf96ac99cfa134c9cb2820e01e91282a06e8c285c17578c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , ceph=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, version=8, distribution-scope=public, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-09-17T01:51:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:10 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:51:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:10 smithi105 ceph-mon[63427]: osdmap e119: 8 total, 7 up, 8 in 2024-09-17T01:51:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:10 smithi105 ceph-mon[63427]: pgmap v75: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:10.644 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.335985064 +0000 UTC m=+0.105732645 container create f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T01:51:10.644 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.246124654 +0000 UTC m=+0.015872260 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:10.644 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.555236505 +0000 UTC m=+0.324984098 container init f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:51:10.644 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.558728501 +0000 UTC m=+0.328476084 container start f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731) 2024-09-17T01:51:10.645 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.646777945 +0000 UTC m=+0.416525538 container attach f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:51:10.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:10 smithi081 podman[91840]: 2024-09-17 01:51:10.744807423 +0000 UTC m=+0.514555022 container died f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T01:51:11.373 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:11 smithi081 ceph-mon[77854]: osdmap e120: 8 total, 7 up, 8 in 2024-09-17T01:51:11.373 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:11 smithi081 ceph-mon[80183]: osdmap e120: 8 total, 7 up, 8 in 2024-09-17T01:51:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:11 smithi105 ceph-mon[63427]: osdmap e120: 8 total, 7 up, 8 in 2024-09-17T01:51:11.640 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 podman[91863]: 2024-09-17 01:51:11.375637839 +0000 UTC m=+0.620113719 container remove f30d2a6aed5c903420dfc4d54420a20a9f3d0868eb5965bea8f3a1b031030b34 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T01:51:11.640 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T01:51:11.640 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 systemd[1]: Stopped Ceph osd.1 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:51:11.640 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.1.service: Consumed 13.891s CPU time. 2024-09-17T01:51:11.640 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 systemd[1]: Starting Ceph osd.1 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:51:11.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 podman[91960]: 2024-09-17 01:51:11.735748493 +0000 UTC m=+0.015937758 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:11.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:11 smithi081 podman[91960]: 2024-09-17 01:51:11.851836382 +0000 UTC m=+0.132025640 container create 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T01:51:12.228 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:12 smithi081 ceph-mon[77854]: pgmap v77: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:51:12.229 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:12 smithi081 ceph-mon[80183]: pgmap v77: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:51:12.229 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 podman[91960]: 2024-09-17 01:51:12.096183142 +0000 UTC m=+0.376372394 container init 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T01:51:12.229 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 podman[91960]: 2024-09-17 01:51:12.09995547 +0000 UTC m=+0.380144735 container start 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-17T01:51:12.229 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 podman[91960]: 2024-09-17 01:51:12.171082072 +0000 UTC m=+0.451271326 container attach 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-17T01:51:12.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:12 smithi105 ceph-mon[63427]: pgmap v77: 161 pgs: 56 peering, 1 stale+active+clean, 104 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:51:12.608 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:51:12.608 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 bash[91960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:51:12.608 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:51:12.608 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:12 smithi081 bash[91960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:51:13.299 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-mon[77854]: Health check failed: Reduced data availability: 10 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-09-17T01:51:13.299 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-mon[80183]: Health check failed: Reduced data availability: 10 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-09-17T01:51:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:13 smithi105 ceph-mon[63427]: Health check failed: Reduced data availability: 10 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-09-17T01:51:13.589 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:13] "GET /metrics HTTP/1.1" 200 38103 "" "Prometheus/2.43.0" 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:51:13.589 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate[91983]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 bash[91960]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T01:51:13.915 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:13 smithi081 podman[91960]: 2024-09-17 01:51:13.61950954 +0000 UTC m=+1.899698803 container died 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T01:51:14.180 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 podman[92115]: 2024-09-17 01:51:14.166805499 +0000 UTC m=+0.541439960 container remove 6a18c900c9b8798e0ca4190e1a4fc53842e2094a6df1bb842b94b76f25ab7216 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T01:51:14.659 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 podman[92175]: 2024-09-17 01:51:14.276704534 +0000 UTC m=+0.015756192 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:14.660 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 podman[92175]: 2024-09-17 01:51:14.375916272 +0000 UTC m=+0.114967966 container create 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:51:14.660 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 podman[92175]: 2024-09-17 01:51:14.570257362 +0000 UTC m=+0.309309025 container init 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-09-17T01:51:14.660 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 podman[92175]: 2024-09-17 01:51:14.573516567 +0000 UTC m=+0.312568225 container start 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=squid-c2ddf97) 2024-09-17T01:51:14.660 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-osd[92194]: -- 172.21.15.81:0/195018723 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f47f803a40 con 0x55f47f7e1c00 2024-09-17T01:51:14.660 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 bash[92175]: 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 2024-09-17T01:51:14.930 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:14 smithi081 systemd[1]: Started Ceph osd.1 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:51:14.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: pgmap v78: 161 pgs: 6 active+undersized, 56 peering, 1 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: pgmap v78: 161 pgs: 6 active+undersized, 56 peering, 1 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:14.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:51:14.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:14.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: pgmap v78: 161 pgs: 6 active+undersized, 56 peering, 1 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/723 objects degraded (0.830%) 2024-09-17T01:51:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 6/723 objects degraded (0.830%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:51:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:16.090 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:16.090 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:15 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:16.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:16.735 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:16 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:51:16.460+0000 7f5bdb8c3640 -1 Falling back to public interface 2024-09-17T01:51:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:16 smithi105 ceph-mon[63427]: pgmap v79: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:16 smithi105 ceph-mon[63427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 16 pgs peering) 2024-09-17T01:51:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:16 smithi081 ceph-mon[80183]: pgmap v79: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:16 smithi081 ceph-mon[80183]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 16 pgs peering) 2024-09-17T01:51:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:16 smithi081 ceph-mon[77854]: pgmap v79: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:16 smithi081 ceph-mon[77854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 16 pgs peering) 2024-09-17T01:51:18.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:18 smithi081 ceph-mon[77854]: pgmap v80: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:18.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:18 smithi081 ceph-mon[80183]: pgmap v80: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:18 smithi105 ceph-mon[63427]: pgmap v80: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:20.096 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:51:19.726+0000 7f5bdb8c3640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T01:51:20.097 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:51:19.757+0000 7f5bdb8c3640 -1 osd.1 118 log_to_monitors true 2024-09-17T01:51:20.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: pgmap v81: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:20.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.351 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[77854]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: pgmap v81: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:20.352 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:20 smithi081 ceph-mon[80183]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: pgmap v81: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:20.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:20.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:20.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:20 smithi105 ceph-mon[63427]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:51:21.340 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:51:21.202+0000 7f5bcf666640 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: osdmap e121: 8 total, 7 up, 8 in 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[77854]: from='osd.1 ' entity='osd.1' 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:51:22.058 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: osdmap e121: 8 total, 7 up, 8 in 2024-09-17T01:51:22.059 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.059 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.059 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:22.059 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:21 smithi081 ceph-mon[80183]: from='osd.1 ' entity='osd.1' 2024-09-17T01:51:22.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:51:22.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: osdmap e121: 8 total, 7 up, 8 in 2024-09-17T01:51:22.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: from='osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:51:22.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:22.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:21 smithi105 ceph-mon[63427]: from='osd.1 ' entity='osd.1' 2024-09-17T01:51:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: pgmap v83: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:51:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608] boot 2024-09-17T01:51:22.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: pgmap v83: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608] boot 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T01:51:22.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:51:22.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:22.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:22.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:22.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:22.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:22 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: pgmap v83: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: osd.1 [v2:172.21.15.81:6810/437248608,v1:172.21.15.81:6811/437248608] boot 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:23.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:22 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.832 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:23] "GET /metrics HTTP/1.1" 200 38243 "" "Prometheus/2.43.0" 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v85: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v86: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:51:23.833 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: Upgrade: Finalizing container_image settings 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:51:23.834 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v85: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v86: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.835 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: Upgrade: Finalizing container_image settings 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.836 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:51:23.837 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.838 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: Upgrade: Complete! 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v87: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v88: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v89: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: pgmap v90: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.839 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[77854]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.840 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: Upgrade: Complete! 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v87: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v88: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v89: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.841 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: pgmap v90: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:23.842 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:23.842 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:23 smithi081 ceph-mon[80183]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v85: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 698 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v86: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 961 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:51:24.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:51:24.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:51:24.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v87: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v88: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v89: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: pgmap v90: 161 pgs: 42 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 224 MiB used, 715 GiB / 715 GiB avail; 101/723 objects degraded (13.970%) 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:24.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:23 smithi105 ceph-mon[63427]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T01:51:25.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:25.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:25.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:25 smithi105 ceph-mon[63427]: pgmap v92: 161 pgs: 20 active+undersized, 7 active+undersized+degraded, 134 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 44/723 objects degraded (6.086%) 2024-09-17T01:51:26.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:26.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:25 smithi081 ceph-mon[77854]: pgmap v92: 161 pgs: 20 active+undersized, 7 active+undersized+degraded, 134 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 44/723 objects degraded (6.086%) 2024-09-17T01:51:26.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:25 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:26.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:25 smithi081 ceph-mon[80183]: pgmap v92: 161 pgs: 20 active+undersized, 7 active+undersized+degraded, 134 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 44/723 objects degraded (6.086%) 2024-09-17T01:51:27.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:26 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:27.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:26 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:27.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:26 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:51:28.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:27 smithi105 ceph-mon[63427]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:28.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:27 smithi081 ceph-mon[77854]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:28.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:27 smithi081 ceph-mon[80183]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:29.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:28 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded) 2024-09-17T01:51:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded) 2024-09-17T01:51:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 7 pgs degraded) 2024-09-17T01:51:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:31.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:30 smithi105 ceph-mon[63427]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:51:31.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:30 smithi081 ceph-mon[77854]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:51:31.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:30 smithi081 ceph-mon[80183]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:51:33.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:32 smithi105 ceph-mon[63427]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:32 smithi081 ceph-mon[77854]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:33.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:32 smithi081 ceph-mon[80183]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:33] "GET /metrics HTTP/1.1" 200 38263 "" "Prometheus/2.43.0" 2024-09-17T01:51:34.408 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:51:34.953 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[77854]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:34.954 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[77854]: from='client.54285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:34.954 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:34.954 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[80183]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:34.954 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[80183]: from='client.54285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:34.954 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:34 smithi105 ceph-mon[63427]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:51:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:34 smithi105 ceph-mon[63427]: from='client.54285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:35.966 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:35.966 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:36.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:36.522 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:51:36.522 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (17m) 17s ago 25m 25.0M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (4m) 2m ago 25m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 17s ago 24m 58.5M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (4m) 2m ago 29m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (13m) 17s ago 31m 500M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (2m) 17s ago 31m 47.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (3m) 2m ago 30m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (2m) 17s ago 30m 37.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (17m) 17s ago 26m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (17m) 2m ago 26m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (55s) 17s ago 29m 43.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (22s) 17s ago 29m 16.5M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (28m) 17s ago 28m 53.1M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (28m) 17s ago 28m 55.9M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:51:36.523 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (28m) 2m ago 28m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (27m) 2m ago 27m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (27m) 2m ago 27m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (26m) 2m ago 26m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (4m) 2m ago 26m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (24m) 17s ago 24m 79.4M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:51:36.524 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (24m) 2m ago 24m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:51:37.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:36 smithi105 ceph-mon[63427]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:51:37.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:36 smithi081 ceph-mon[77854]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:51:37.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:36 smithi081 ceph-mon[80183]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:51:37.313 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-17T01:51:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:37 smithi105 ceph-mon[63427]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:38.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:37 smithi081 ceph-mon[77854]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:38.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:37 smithi081 ceph-mon[80183]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:38.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:38 smithi081 ceph-mon[77854]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:38.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:38 smithi081 ceph-mon[80183]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:39.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:38 smithi105 ceph-mon[63427]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:39.560 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:51:40.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:39 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4010791984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:40.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:39 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4010791984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:40.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:39 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4010791984' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:40.320 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-09-17T01:51:41.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:40 smithi105 ceph-mon[63427]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:41.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:40 smithi081 ceph-mon[77854]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:41.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:40 smithi081 ceph-mon[80183]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:43.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:42 smithi105 ceph-mon[63427]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:43.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:42 smithi081 ceph-mon[77854]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:43.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:42 smithi081 ceph-mon[80183]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:43] "GET /metrics HTTP/1.1" 200 38263 "" "Prometheus/2.43.0" 2024-09-17T01:51:43.844 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:51:44.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:43 smithi105 ceph-mon[63427]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:44.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:44.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:43 smithi081 ceph-mon[77854]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:44.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:44.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:43 smithi081 ceph-mon[80183]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:44.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:44.586 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:51:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:44 smithi105 ceph-mon[63427]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:44 smithi081 ceph-mon[77854]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:44 smithi081 ceph-mon[80183]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:46.130 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:45 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:46.130 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:45 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:45 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:46.597 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:51:46.597 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:51:46.597 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:51:46.597 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:51:46.597 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:51:46.598 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:51:46.598 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:51:46.598 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:51:46.598 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:51:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:46 smithi081 ceph-mon[77854]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:46 smithi081 ceph-mon[80183]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:47.216 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:51:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:46 smithi105 ceph-mon[63427]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:48.174 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:47 smithi081 ceph-mon[77854]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:48.174 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:47 smithi081 ceph-mon[80183]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:47 smithi105 ceph-mon[63427]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:49.012 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:48 smithi081 ceph-mon[77854]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:49.012 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:48 smithi081 ceph-mon[80183]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:49.318 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:51:49.318 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:51:49.318 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:51:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:48 smithi105 ceph-mon[63427]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:49.925 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-09-17T01:51:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:49 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3078880422' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:51:50.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:49 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3078880422' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:51:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:49 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3078880422' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:51:51.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:50 smithi081 ceph-mon[77854]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:51.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:50 smithi081 ceph-mon[80183]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:50 smithi105 ceph-mon[63427]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:53.131 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:52 smithi081 ceph-mon[77854]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:53.131 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:52 smithi081 ceph-mon[77854]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:53.132 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:52 smithi081 ceph-mon[80183]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:53.132 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:52 smithi081 ceph-mon[80183]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:53.163 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:53.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:52 smithi105 ceph-mon[63427]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:53.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:52 smithi105 ceph-mon[63427]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:53.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:51:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:51:53] "GET /metrics HTTP/1.1" 200 38262 "" "Prometheus/2.43.0" 2024-09-17T01:51:53.826 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:51:54.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[77854]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:54 smithi081 ceph-mon[80183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:51:54.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 0 op/s 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:54.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:54 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.143 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:51:56.144 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.145 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:55 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:56.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:56.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:56.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:55 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mgr 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: Upgrade: osd.2 is safe to restart 2024-09-17T01:51:57.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[77854]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mgr 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: Upgrade: osd.2 is safe to restart 2024-09-17T01:51:57.143 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:56 smithi081 ceph-mon[80183]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:51:57.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:51:57.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:51:57.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:51:57.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: Upgrade: osd.2 is safe to restart 2024-09-17T01:51:57.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:56 smithi105 ceph-mon[63427]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:57.960 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:51:58.426 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: Upgrade: Updating osd.2 2024-09-17T01:51:58.426 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:58.426 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:51:58.426 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: Deploying daemon osd.2 on smithi081 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[80183]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: Upgrade: Updating osd.2 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: Deploying daemon osd.2 on smithi081 2024-09-17T01:51:58.427 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:58.428 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:58 smithi081 ceph-mon[77854]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (17m) 39s ago 25m 25.0M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (4m) 2m ago 25m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 39s ago 24m 58.5M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (4m) 2m ago 30m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (14m) 39s ago 31m 500M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (2m) 39s ago 32m 47.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (3m) 2m ago 30m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (2m) 39s ago 30m 37.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:51:58.623 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (18m) 39s ago 26m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (17m) 2m ago 26m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (77s) 39s ago 30m 43.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (44s) 39s ago 29m 16.5M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (29m) 39s ago 29m 53.1M 2992M 17.2.0 e1d6a67b021e c3e05eebc991 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (28m) 39s ago 28m 55.9M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (28m) 2m ago 28m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (28m) 2m ago 28m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (27m) 2m ago 27m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (27m) 2m ago 27m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (5m) 2m ago 26m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:51:58.624 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (25m) 39s ago 25m 79.4M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:51:58.625 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (25m) 2m ago 25m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:51:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: Upgrade: Updating osd.2 2024-09-17T01:51:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:51:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:51:58.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:51:58.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: Deploying daemon osd.2 on smithi081 2024-09-17T01:51:58.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:51:58.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:58 smithi105 ceph-mon[63427]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:51:59.038 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:51:59.039 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:51:59.314 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[77854]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[77854]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3259960624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[80183]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[80183]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:59.315 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:51:59 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3259960624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:51:59.391 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:51:59.391 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:51:59.391 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:51:59.391 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-09-17T01:51:59.391 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:51:59.392 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "2/8 daemons upgraded", 2024-09-17T01:51:59.392 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:51:59.392 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:51:59.392 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:51:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:59 smithi105 ceph-mon[63427]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:59 smithi105 ceph-mon[63427]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:51:59.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:51:59.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:51:59 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3259960624' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:00.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-mon[77854]: from='client.54351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:00.515 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-mon[77854]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:52:00.515 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-mon[80183]: from='client.54351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:00.515 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-mon[80183]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:52:00.516 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:00 smithi081 systemd[1]: Stopping Ceph osd.2 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:52:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:00 smithi105 ceph-mon[63427]: from='client.54351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:00 smithi105 ceph-mon[63427]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 600 B/s rd, 0 op/s 2024-09-17T01:52:00.930 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[36841]: 2024-09-17T01:52:00.515+0000 7f4985e5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:52:00.930 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[36841]: 2024-09-17T01:52:00.515+0000 7f4985e5d700 -1 osd.2 123 *** Got signal Terminated *** 2024-09-17T01:52:00.930 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[36841]: 2024-09-17T01:52:00.515+0000 7f4985e5d700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:52:01.586 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:01 smithi081 ceph-mon[77854]: osd.2 marked itself down and dead 2024-09-17T01:52:01.587 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:01 smithi081 ceph-mon[80183]: osd.2 marked itself down and dead 2024-09-17T01:52:01.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:01 smithi105 ceph-mon[63427]: osd.2 marked itself down and dead 2024-09-17T01:52:01.850 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:01 smithi081 podman[97993]: 2024-09-17 01:52:01.589031053 +0000 UTC m=+1.170698596 container died c3e05eebc991fca90ece59c281f09ff7025f34b8c8ffbd1a266305bd6bb75289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.expose-services=) 2024-09-17T01:52:02.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:01 smithi081 podman[97993]: 2024-09-17 01:52:01.96644784 +0000 UTC m=+1.548115375 container cleanup c3e05eebc991fca90ece59c281f09ff7025f34b8c8ffbd1a266305bd6bb75289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_CLEAN=True, ceph=True, architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-09-17T01:52:02.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:01 smithi081 bash[97993]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2 2024-09-17T01:52:02.579 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:02.579 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[80183]: osdmap e124: 8 total, 7 up, 8 in 2024-09-17T01:52:02.579 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[80183]: pgmap v113: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:02.580 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:02.580 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[77854]: osdmap e124: 8 total, 7 up, 8 in 2024-09-17T01:52:02.580 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:02 smithi081 ceph-mon[77854]: pgmap v113: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:02.580 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98009]: 2024-09-17 01:52:02.251243427 +0000 UTC m=+0.659192008 container remove c3e05eebc991fca90ece59c281f09ff7025f34b8c8ffbd1a266305bd6bb75289 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, architecture=x86_64, ceph=True, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8) 2024-09-17T01:52:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:02 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:02 smithi105 ceph-mon[63427]: osdmap e124: 8 total, 7 up, 8 in 2024-09-17T01:52:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:02 smithi105 ceph-mon[63427]: pgmap v113: 161 pgs: 12 stale+active+clean, 149 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:02.835 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98097]: 2024-09-17 01:52:02.484781374 +0000 UTC m=+0.016282283 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:02.836 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98097]: 2024-09-17 01:52:02.585656691 +0000 UTC m=+0.117157669 container create 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2) 2024-09-17T01:52:02.836 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98097]: 2024-09-17 01:52:02.838409367 +0000 UTC m=+0.369910276 container init 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T01:52:03.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98097]: 2024-09-17 01:52:02.841729003 +0000 UTC m=+0.373229918 container start 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T01:52:03.181 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:02 smithi081 podman[98097]: 2024-09-17 01:52:02.913195195 +0000 UTC m=+0.444696118 container attach 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:52:03.181 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 podman[98097]: 2024-09-17 01:52:03.029663031 +0000 UTC m=+0.561163948 container died 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T01:52:03.481 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:03 smithi081 ceph-mon[77854]: osdmap e125: 8 total, 7 up, 8 in 2024-09-17T01:52:03.481 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:03 smithi081 ceph-mon[80183]: osdmap e125: 8 total, 7 up, 8 in 2024-09-17T01:52:03.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:03 smithi105 ceph-mon[63427]: osdmap e125: 8 total, 7 up, 8 in 2024-09-17T01:52:03.797 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:03] "GET /metrics HTTP/1.1" 200 38260 "" "Prometheus/2.43.0" 2024-09-17T01:52:03.797 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 podman[98120]: 2024-09-17 01:52:03.567070354 +0000 UTC m=+0.531636448 container remove 74ff4d9fad001384ea3349a9f087aedc42a0514741892704ac953a292e715f3e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:52:03.797 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T01:52:03.797 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 systemd[1]: Stopped Ceph osd.2 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:52:03.797 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.2.service: Consumed 11.567s CPU time. 2024-09-17T01:52:04.118 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:03 smithi081 systemd[1]: Starting Ceph osd.2 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:52:04.118 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 podman[98218]: 2024-09-17 01:52:04.010254682 +0000 UTC m=+0.112413499 container create 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T01:52:04.118 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 podman[98218]: 2024-09-17 01:52:03.913793841 +0000 UTC m=+0.015952629 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:04.383 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:04 smithi081 ceph-mon[77854]: pgmap v115: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:04.383 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:04 smithi081 ceph-mon[80183]: pgmap v115: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:04.383 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 podman[98218]: 2024-09-17 01:52:04.254571839 +0000 UTC m=+0.356730611 container init 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97) 2024-09-17T01:52:04.384 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 podman[98218]: 2024-09-17 01:52:04.258073652 +0000 UTC m=+0.360232432 container start 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True) 2024-09-17T01:52:04.384 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 podman[98218]: 2024-09-17 01:52:04.346073269 +0000 UTC m=+0.448232042 container attach 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-17T01:52:04.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:04 smithi105 ceph-mon[63427]: pgmap v115: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:04.680 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:04.681 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 bash[98218]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:04.681 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:04.681 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:04 smithi081 bash[98218]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-mon[77854]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T01:52:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-mon[80183]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T01:52:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:05.431 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:52:05.431 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:52:05.431 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:52:05.431 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:52:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:05 smithi105 ceph-mon[63427]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T01:52:05.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:05 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:52:05.772 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate[98234]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T01:52:05.773 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 bash[98218]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T01:52:05.773 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:05 smithi081 podman[98218]: 2024-09-17 01:52:05.468335012 +0000 UTC m=+1.570493795 container died 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-17T01:52:06.384 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:06 smithi081 ceph-mon[77854]: pgmap v116: 161 pgs: 22 active+undersized, 7 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 39/723 objects degraded (5.394%) 2024-09-17T01:52:06.384 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:06 smithi081 ceph-mon[80183]: pgmap v116: 161 pgs: 22 active+undersized, 7 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 39/723 objects degraded (5.394%) 2024-09-17T01:52:06.384 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 podman[98372]: 2024-09-17 01:52:06.05010222 +0000 UTC m=+0.572146910 container remove 247af3e4db33e5aa44c655815e0c0f841d4a37062aee8d718738d20564b82d93 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T01:52:06.385 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 podman[98432]: 2024-09-17 01:52:06.159329108 +0000 UTC m=+0.015463683 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:06.385 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 podman[98432]: 2024-09-17 01:52:06.259239518 +0000 UTC m=+0.115374129 container create 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97) 2024-09-17T01:52:06.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:06 smithi105 ceph-mon[63427]: pgmap v116: 161 pgs: 22 active+undersized, 7 peering, 2 stale+active+clean, 11 active+undersized+degraded, 119 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 39/723 objects degraded (5.394%) 2024-09-17T01:52:06.680 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 podman[98432]: 2024-09-17 01:52:06.580053621 +0000 UTC m=+0.436188201 container init 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-17T01:52:06.680 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 podman[98432]: 2024-09-17 01:52:06.583266543 +0000 UTC m=+0.439401104 container start 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97) 2024-09-17T01:52:06.681 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 ceph-osd[98451]: -- 172.21.15.81:0/2995758575 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5588bad1da40 con 0x5588bbae4000 2024-09-17T01:52:07.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 bash[98432]: 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 2024-09-17T01:52:07.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:06 smithi081 systemd[1]: Started Ceph osd.2 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 39/723 objects degraded (5.394%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 39/723 objects degraded (5.394%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.461 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:07 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:07 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 39/723 objects degraded (5.394%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:07.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:07 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:07 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:07.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:07 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:08.476 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:08 smithi081 ceph-mon[77854]: pgmap v117: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:08.477 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:08 smithi081 ceph-mon[80183]: pgmap v117: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:08.477 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:08 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:52:08.477+0000 7fb94dced640 -1 Falling back to public interface 2024-09-17T01:52:08.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:08 smithi105 ceph-mon[63427]: pgmap v117: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:09.571 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:09 smithi081 ceph-mon[80183]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T01:52:09.571 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:09 smithi081 ceph-mon[77854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T01:52:09.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:09 smithi105 ceph-mon[63427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T01:52:10.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:10 smithi105 ceph-mon[63427]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-09-17T01:52:10.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:10 smithi081 ceph-mon[77854]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-09-17T01:52:10.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:10 smithi081 ceph-mon[80183]: pgmap v118: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-09-17T01:52:12.396 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:52:11.996+0000 7fb94dced640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T01:52:12.396 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:52:12.024+0000 7fb94dced640 -1 osd.2 123 log_to_monitors true 2024-09-17T01:52:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:12.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:52:12.682 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:12 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: pgmap v119: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:12.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:12.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:52:12.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:12 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:13.583 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:13] "GET /metrics HTTP/1.1" 200 38260 "" "Prometheus/2.43.0" 2024-09-17T01:52:13.583 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:52:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:52:13.473+0000 7fb941a90640 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:52:14.317 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:52:14.317 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: osdmap e126: 8 total, 7 up, 8 in 2024-09-17T01:52:14.317 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:52:14.317 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: osdmap e126: 8 total, 7 up, 8 in 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:14.318 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:14.319 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: osdmap e126: 8 total, 7 up, 8 in 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411]' entity='osd.2' 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: pgmap v121: 161 pgs: 33 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:52:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[77854]: osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411] boot 2024-09-17T01:52:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[77854]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[80183]: osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411] boot 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[80183]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:52:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:15 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:15 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:52:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:15 smithi105 ceph-mon[63427]: osd.2 [v2:172.21.15.81:6818/600115411,v1:172.21.15.81:6819/600115411] boot 2024-09-17T01:52:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:15 smithi105 ceph-mon[63427]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T01:52:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:15 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:52:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v123: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v124: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.387 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:52:16.388 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:52:16.389 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v125: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v126: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v127: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: pgmap v128: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 5 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T01:52:16.390 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:16 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v123: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v124: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: Upgrade: Finalizing container_image settings 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:52:16.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.433 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v123: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v124: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.434 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: Upgrade: Finalizing container_image settings 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:52:16.435 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:52:16.436 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: Upgrade: Complete! 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v125: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v126: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.437 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v127: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: pgmap v128: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 5 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:52:16.438 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.439 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: Upgrade: Complete! 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v125: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v126: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:16.440 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:16.441 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v127: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.441 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: pgmap v128: 161 pgs: 25 peering, 18 active+undersized, 4 active+undersized+degraded, 114 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 5 op/s; 29/723 objects degraded (4.011%) 2024-09-17T01:52:16.441 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:16.441 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T01:52:16.441 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:16 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:52:18.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:18 smithi105 ceph-mon[63427]: pgmap v130: 161 pgs: 25 peering, 11 active+undersized, 1 active+undersized+degraded, 124 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-17T01:52:18.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:18 smithi081 ceph-mon[80183]: pgmap v130: 161 pgs: 25 peering, 11 active+undersized, 1 active+undersized+degraded, 124 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-17T01:52:18.681 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:18 smithi081 ceph-mon[77854]: pgmap v130: 161 pgs: 25 peering, 11 active+undersized, 1 active+undersized+degraded, 124 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-09-17T01:52:20.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:20.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:20.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:21.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:20 smithi105 ceph-mon[63427]: pgmap v131: 161 pgs: 25 peering, 136 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:52:21.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:20 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-17T01:52:21.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:20 smithi081 ceph-mon[80183]: pgmap v131: 161 pgs: 25 peering, 136 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:52:21.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:20 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-17T01:52:21.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:20 smithi081 ceph-mon[77854]: pgmap v131: 161 pgs: 25 peering, 136 active+clean; 585 KiB data, 157 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:52:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:20 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded) 2024-09-17T01:52:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:22 smithi081 ceph-mon[77854]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:52:23.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:22 smithi081 ceph-mon[80183]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:52:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:22 smithi105 ceph-mon[63427]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:52:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:23] "GET /metrics HTTP/1.1" 200 38263 "" "Prometheus/2.43.0" 2024-09-17T01:52:25.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:24 smithi081 ceph-mon[77854]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:25.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:24 smithi081 ceph-mon[80183]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:25.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:24 smithi105 ceph-mon[63427]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:52:26.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:26.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:25 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:26.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:27.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:26 smithi081 ceph-mon[77854]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:26 smithi081 ceph-mon[80183]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:26 smithi105 ceph-mon[63427]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[77854]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-09-17T01:52:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[80183]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-09-17T01:52:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:29.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:28 smithi105 ceph-mon[63427]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-09-17T01:52:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:30.382 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:52:30.996 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:30 smithi081 ceph-mon[77854]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:30.996 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:30 smithi081 ceph-mon[77854]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:30.996 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:30 smithi081 ceph-mon[80183]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:30.996 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:30 smithi081 ceph-mon[80183]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:30 smithi105 ceph-mon[63427]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:30 smithi105 ceph-mon[63427]: from='client.64330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:32.280 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:52:32.280 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (18m) 20s ago 26m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (5m) 3m ago 26m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 20s ago 25m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (5m) 3m ago 30m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (14m) 20s ago 32m 501M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (3m) 20s ago 32m 50.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (4m) 3m ago 31m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (3m) 20s ago 31m 39.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (18m) 20s ago 27m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (18m) 3m ago 27m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (111s) 20s ago 30m 44.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (77s) 20s ago 30m 56.5M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (26s) 20s ago 29m 11.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:52:32.281 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (29m) 20s ago 29m 57.6M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (29m) 3m ago 29m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (28m) 3m ago 28m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (28m) 3m ago 28m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (27m) 3m ago 27m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (5m) 3m ago 27m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (25m) 20s ago 25m 79.6M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:52:32.282 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (25m) 3m ago 25m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:52:32.923 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-17T01:52:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:32 smithi081 ceph-mon[77854]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:33.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:32 smithi081 ceph-mon[80183]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:32 smithi105 ceph-mon[63427]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:33.913 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:33] "GET /metrics HTTP/1.1" 200 38276 "" "Prometheus/2.43.0" 2024-09-17T01:52:34.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:33 smithi081 ceph-mon[77854]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:34.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:33 smithi081 ceph-mon[80183]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:33 smithi105 ceph-mon[63427]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:34.966 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:52:35.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:34 smithi081 ceph-mon[77854]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:35.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:34 smithi081 ceph-mon[80183]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:34 smithi105 ceph-mon[63427]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:35.563 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-09-17T01:52:36.167 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:36.167 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:35 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4053526697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:36.167 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:36.167 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:35 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4053526697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:35 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4053526697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:36.958 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:36 smithi081 ceph-mon[77854]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:36.959 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:36 smithi081 ceph-mon[80183]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:36 smithi105 ceph-mon[63427]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:38.759 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:52:39.042 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:38 smithi081 ceph-mon[77854]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:39.043 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:38 smithi081 ceph-mon[77854]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:39.043 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:38 smithi081 ceph-mon[80183]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:39.043 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:38 smithi081 ceph-mon[80183]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:38 smithi105 ceph-mon[63427]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:38 smithi105 ceph-mon[63427]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:39.391 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:52:41.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:40 smithi081 ceph-mon[80183]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:41.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:40 smithi081 ceph-mon[77854]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:52:41.366 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:52:41.367 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:52:41.367 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:52:41.367 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:52:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:40 smithi105 ceph-mon[63427]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:41.997 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:52:43.105 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:42 smithi081 ceph-mon[77854]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:43.112 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:42 smithi081 ceph-mon[77854]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:43.113 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:42 smithi081 ceph-mon[80183]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:43.113 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:42 smithi081 ceph-mon[80183]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:42 smithi105 ceph-mon[63427]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:42 smithi105 ceph-mon[63427]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:43] "GET /metrics HTTP/1.1" 200 38276 "" "Prometheus/2.43.0" 2024-09-17T01:52:43.895 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:52:43.895 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:52:43.895 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:52:44.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:44.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:43 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2124223769' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:52:44.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:44.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:43 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2124223769' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:52:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:43 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2124223769' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:52:44.487 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-09-17T01:52:45.024 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:44 smithi081 ceph-mon[77854]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:45.024 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:44 smithi081 ceph-mon[80183]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:45.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:44 smithi105 ceph-mon[63427]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:46.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:45 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:45 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:45 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:46.955 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:46 smithi081 ceph-mon[77854]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:46.955 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:46 smithi081 ceph-mon[80183]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:46 smithi105 ceph-mon[63427]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:47.647 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[77854]: from='client.54399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[80183]: from='client.54399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:48.377 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.378 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:48.378 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:48.378 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:47 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:47 smithi105 ceph-mon[63427]: from='client.54399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:47 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:47 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:47 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:52:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:47 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:48.398 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[77854]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[77854]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[77854]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[77854]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[77854]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[80183]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:49.219 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[80183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:49.220 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[80183]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-17T01:52:49.220 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[80183]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:49.220 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:48 smithi081 ceph-mon[80183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:48 smithi105 ceph-mon[63427]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:52:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:48 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:48 smithi105 ceph-mon[63427]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-09-17T01:52:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:48 smithi105 ceph-mon[63427]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:48 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mgr 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.660 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: Upgrade: osd.3 is safe to restart 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[80183]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:52:50.661 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mgr 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.662 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: Upgrade: osd.3 is safe to restart 2024-09-17T01:52:50.663 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:50 smithi081 ceph-mon[77854]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:52:50.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:52:50.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: Upgrade: osd.3 is safe to restart 2024-09-17T01:52:50.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:50 smithi105 ceph-mon[63427]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:52.543 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:52:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: Upgrade: Updating osd.3 2024-09-17T01:52:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:52:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:52 smithi105 ceph-mon[63427]: Deploying daemon osd.3 on smithi081 2024-09-17T01:52:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: Upgrade: Updating osd.3 2024-09-17T01:52:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:52:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[77854]: Deploying daemon osd.3 on smithi081 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: Upgrade: Updating osd.3 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:52:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:52 smithi081 ceph-mon[80183]: Deploying daemon osd.3 on smithi081 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (18m) 41s ago 26m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (5m) 3m ago 26m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 41s ago 25m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (5m) 3m ago 31m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (15m) 41s ago 32m 501M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (3m) 41s ago 33m 50.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:52:53.207 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (4m) 3m ago 31m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (3m) 41s ago 31m 39.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (18m) 41s ago 27m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (18m) 3m ago 27m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (2m) 41s ago 30m 44.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (98s) 41s ago 30m 56.5M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (47s) 41s ago 30m 11.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (29m) 41s ago 29m 57.6M 2992M 17.2.0 e1d6a67b021e b991f6580552 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (29m) 3m ago 29m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (29m) 3m ago 29m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:52:53.208 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (28m) 3m ago 28m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:52:53.209 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (28m) 3m ago 28m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:52:53.209 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (5m) 3m ago 27m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:52:53.209 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (26m) 41s ago 26m 79.6M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:52:53.209 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (26m) 3m ago 26m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:52:53.614 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:52:53.615 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:52:53.754 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:52:53] "GET /metrics HTTP/1.1" 200 38273 "" "Prometheus/2.43.0" 2024-09-17T01:52:53.959 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:52:53.959 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "3/8 daemons upgraded", 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:52:53.960 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:52:54.015 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[77854]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.016 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[77854]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.016 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3469957069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:54.016 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[80183]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.016 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[80183]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.016 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:53 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3469957069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:53 smithi105 ceph-mon[63427]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:53 smithi105 ceph-mon[63427]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:54.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:53 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3469957069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:52:55.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[77854]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[77854]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:55.059 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[77854]: from='client.54429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.060 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[80183]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.060 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[80183]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:55.060 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:54 smithi081 ceph-mon[80183]: from='client.54429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:54 smithi105 ceph-mon[63427]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:54 smithi105 ceph-mon[63427]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 615 B/s rd, 0 op/s 2024-09-17T01:52:55.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:54 smithi105 ceph-mon[63427]: from='client.54429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:52:55.324 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:55 smithi081 systemd[1]: Stopping Ceph osd.3 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:52:55.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[39796]: 2024-09-17T01:52:55.325+0000 7f7ed9c66700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:52:55.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[39796]: 2024-09-17T01:52:55.325+0000 7f7ed9c66700 -1 osd.3 128 *** Got signal Terminated *** 2024-09-17T01:52:55.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[39796]: 2024-09-17T01:52:55.325+0000 7f7ed9c66700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:52:56.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:55 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:56.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:55 smithi105 ceph-mon[63427]: osd.3 marked itself down and dead 2024-09-17T01:52:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:56.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-mon[77854]: osd.3 marked itself down and dead 2024-09-17T01:52:56.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:52:56.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:55 smithi081 ceph-mon[80183]: osd.3 marked itself down and dead 2024-09-17T01:52:56.666 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:56 smithi081 podman[104487]: 2024-09-17 01:52:56.415651508 +0000 UTC m=+1.210853274 container died b991f65805524eb759f20044c517110bf3ddd662e61d2572fab89a00efea8058 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, version=8, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T01:52:56.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[77854]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:56.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:56.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[77854]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T01:52:56.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[80183]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:56.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:56.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:56 smithi081 ceph-mon[80183]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T01:52:56.931 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:56 smithi081 podman[104487]: 2024-09-17 01:52:56.859293373 +0000 UTC m=+1.654495134 container cleanup b991f65805524eb759f20044c517110bf3ddd662e61d2572fab89a00efea8058 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-09-17T01:52:56.931 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:56 smithi081 bash[104487]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3 2024-09-17T01:52:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:56 smithi105 ceph-mon[63427]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:52:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:56 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:52:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:56 smithi105 ceph-mon[63427]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T01:52:57.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104502]: 2024-09-17 01:52:57.044225927 +0000 UTC m=+0.620798135 container remove b991f65805524eb759f20044c517110bf3ddd662e61d2572fab89a00efea8058 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, name=centos-stream, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git) 2024-09-17T01:52:57.762 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.367409721 +0000 UTC m=+0.016211506 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:57.762 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.46796883 +0000 UTC m=+0.116770603 container create 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T01:52:57.762 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.673140748 +0000 UTC m=+0.321942528 container init 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-17T01:52:57.762 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.677221158 +0000 UTC m=+0.326022939 container start 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:52:57.762 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.76461293 +0000 UTC m=+0.413414722 container attach 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T01:52:58.165 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:57 smithi081 ceph-mon[77854]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T01:52:58.166 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:57 smithi081 podman[104592]: 2024-09-17 01:52:57.862939759 +0000 UTC m=+0.511741549 container died 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T01:52:58.166 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:57 smithi081 ceph-mon[80183]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T01:52:58.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:57 smithi105 ceph-mon[63427]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T01:52:58.894 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 podman[104615]: 2024-09-17 01:52:58.493892158 +0000 UTC m=+0.625244153 container remove 8321ac599d3bbfbd45aa190051d6b36fa027e9d6882cccae93cd3c1c38a687b2 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-17T01:52:58.895 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T01:52:58.895 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 systemd[1]: Stopped Ceph osd.3 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:52:58.895 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.3.service: Consumed 15.913s CPU time. 2024-09-17T01:52:58.895 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 systemd[1]: Starting Ceph osd.3 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:52:59.170 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[77854]: pgmap v154: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[77854]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[80183]: pgmap v154: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[80183]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:59.171 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:59.171 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 podman[104714]: 2024-09-17 01:52:58.833221723 +0000 UTC m=+0.016551633 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:52:59.172 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:58 smithi081 podman[104714]: 2024-09-17 01:52:58.953893313 +0000 UTC m=+0.137223211 container create cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T01:52:59.172 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 podman[104714]: 2024-09-17 01:52:59.173201552 +0000 UTC m=+0.356531459 container init cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:52:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:58 smithi105 ceph-mon[63427]: pgmap v154: 161 pgs: 14 peering, 21 stale+active+clean, 126 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:52:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:58 smithi105 ceph-mon[63427]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T01:52:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:52:59.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:52:59.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 podman[104714]: 2024-09-17 01:52:59.176660409 +0000 UTC m=+0.359990307 container start cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97) 2024-09-17T01:52:59.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 podman[104714]: 2024-09-17 01:52:59.281590021 +0000 UTC m=+0.464919923 container attach cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T01:52:59.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:59.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 bash[104714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:59.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:52:59.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 bash[104714]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:53:00.067 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:53:00.067 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:53:00.067 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:53:00.067 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 bash[104714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:53:00.067 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:53:00.067 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:52:59 smithi081 bash[104714]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:53:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:52:59 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:53:00.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:53:00.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 bash[104714]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:53:00.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:53:00.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 bash[104714]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 bash[104714]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 bash[104714]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate[104729]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 bash[104714]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T01:53:00.431 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 podman[104714]: 2024-09-17 01:53:00.097777224 +0000 UTC m=+1.281107130 container died cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:53:01.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-mon[77854]: pgmap v155: 161 pgs: 5 active+undersized, 14 peering, 20 stale+active+clean, 1 active+undersized+degraded, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 1/723 objects degraded (0.138%) 2024-09-17T01:53:01.136 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:00 smithi081 ceph-mon[80183]: pgmap v155: 161 pgs: 5 active+undersized, 14 peering, 20 stale+active+clean, 1 active+undersized+degraded, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 1/723 objects degraded (0.138%) 2024-09-17T01:53:01.136 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:00 smithi081 podman[104861]: 2024-09-17 01:53:00.79086539 +0000 UTC m=+0.687080274 container remove cb91997d63c49f2241e6554e4a2d74ad123ba99007f96aa3c1b4f5b4b2fdf121 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:53:01.136 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 podman[104921]: 2024-09-17 01:53:00.90372868 +0000 UTC m=+0.016004152 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:53:01.136 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 podman[104921]: 2024-09-17 01:53:01.011973543 +0000 UTC m=+0.124249002 container create 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T01:53:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:00 smithi105 ceph-mon[63427]: pgmap v155: 161 pgs: 5 active+undersized, 14 peering, 20 stale+active+clean, 1 active+undersized+degraded, 121 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 1/723 objects degraded (0.138%) 2024-09-17T01:53:01.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 podman[104921]: 2024-09-17 01:53:01.230839193 +0000 UTC m=+0.343114662 container init 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:53:01.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 podman[104921]: 2024-09-17 01:53:01.234519404 +0000 UTC m=+0.346794866 container start 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T01:53:01.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 ceph-osd[104940]: -- 172.21.15.81:0/1147938912 <== mon.0 v2:172.21.15.81:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563310c6fa40 con 0x563311a36400 2024-09-17T01:53:01.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 bash[104921]: 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af 2024-09-17T01:53:01.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:01 smithi081 systemd[1]: Started Ceph osd.3 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:53:02.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:02 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:02 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:02 smithi105 ceph-mon[63427]: pgmap v156: 161 pgs: 30 active+undersized, 14 peering, 22 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 71/723 objects degraded (9.820%) 2024-09-17T01:53:02.645 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.645 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.645 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[77854]: pgmap v156: 161 pgs: 30 active+undersized, 14 peering, 22 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 71/723 objects degraded (9.820%) 2024-09-17T01:53:02.646 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.646 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:02.646 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-mon[80183]: pgmap v156: 161 pgs: 30 active+undersized, 14 peering, 22 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 71/723 objects degraded (9.820%) 2024-09-17T01:53:02.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:02 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:53:02.646+0000 7f8937677640 -1 Falling back to public interface 2024-09-17T01:53:03.756 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:03] "GET /metrics HTTP/1.1" 200 38272 "" "Prometheus/2.43.0" 2024-09-17T01:53:04.086 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:03 smithi081 ceph-mon[77854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T01:53:04.086 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:03 smithi081 ceph-mon[80183]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T01:53:04.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:03 smithi105 ceph-mon[63427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T01:53:05.088 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:04 smithi081 ceph-mon[77854]: pgmap v157: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:05.089 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:04 smithi081 ceph-mon[80183]: pgmap v157: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:05.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:04 smithi105 ceph-mon[63427]: pgmap v157: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:06.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:05 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:06.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:05 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:05 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:06.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:53:06.187+0000 7f8937677640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T01:53:06.680 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:53:06.216+0000 7f8937677640 -1 osd.3 128 log_to_monitors true 2024-09-17T01:53:07.096 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:53:06.852+0000 7f892f422640 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:53:07.096 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: pgmap v158: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:07.096 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: pgmap v158: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.098 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.099 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: pgmap v158: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:08.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:07 smithi105 ceph-mon[63427]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:53:08.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:07 smithi105 ceph-mon[63427]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T01:53:08.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:07 smithi105 ceph-mon[63427]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:08.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:07 smithi105 ceph-mon[63427]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:08.137 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[77854]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:53:08.137 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[77854]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T01:53:08.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[77854]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:08.138 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[77854]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:08.138 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[80183]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:53:08.138 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[80183]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T01:53:08.138 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[80183]: from='osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:08.139 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:07 smithi081 ceph-mon[80183]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi081", "root=default"]}]: dispatch 2024-09-17T01:53:09.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:08 smithi105 ceph-mon[63427]: pgmap v160: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:09.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:08 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:09.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:08 smithi105 ceph-mon[63427]: osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495] boot 2024-09-17T01:53:09.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:08 smithi105 ceph-mon[63427]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T01:53:09.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:08 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:53:09.147 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[77854]: pgmap v160: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:09.147 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:09.147 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[77854]: osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495] boot 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[77854]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[80183]: pgmap v160: 161 pgs: 39 active+undersized, 27 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[80183]: osd.3 [v2:172.21.15.81:6826/2021071495,v1:172.21.15.81:6827/2021071495] boot 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[80183]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T01:53:09.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:08 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:53:09.849 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:53:09.660+0000 7f469c54b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:53:10.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T01:53:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:10.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T01:53:10.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:11.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:10 smithi105 ceph-mon[63427]: pgmap v163: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:10 smithi105 ceph-mon[63427]: pgmap v164: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:10 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:10 smithi105 ceph-mon[63427]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:53:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[80183]: pgmap v163: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[80183]: pgmap v164: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[80183]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:53:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[77854]: pgmap v163: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 870 B/s rd, 0 op/s; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[77854]: pgmap v164: 161 pgs: 5 peering, 35 active+undersized, 26 active+undersized+degraded, 95 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:53:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:10 smithi081 ceph-mon[77854]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:53:12.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:11 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:12.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:11 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:12.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:11 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 96/723 objects degraded (13.278%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:13.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:12 smithi105 ceph-mon[63427]: pgmap v165: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-09-17T01:53:13.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:12 smithi081 ceph-mon[77854]: pgmap v165: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-09-17T01:53:13.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:12 smithi081 ceph-mon[80183]: pgmap v165: 161 pgs: 5 peering, 21 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 58/723 objects degraded (8.022%) 2024-09-17T01:53:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:13] "GET /metrics HTTP/1.1" 200 38272 "" "Prometheus/2.43.0" 2024-09-17T01:53:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:14 smithi105 ceph-mon[63427]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:14 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-09-17T01:53:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[77854]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[80183]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:14 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded) 2024-09-17T01:53:16.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:16.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:15 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:16 smithi105 ceph-mon[63427]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:16 smithi081 ceph-mon[77854]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:16 smithi081 ceph-mon[80183]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:18 smithi105 ceph-mon[63427]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:18 smithi081 ceph-mon[77854]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:18 smithi081 ceph-mon[80183]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:20.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:20.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:21.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:20 smithi081 ceph-mon[77854]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:21.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:20 smithi081 ceph-mon[80183]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:20 smithi105 ceph-mon[63427]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:22 smithi081 ceph-mon[77854]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:23.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:22 smithi081 ceph-mon[80183]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:22 smithi105 ceph-mon[63427]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:53:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:23] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-09-17T01:53:24.325 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:53:24.991 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:53:24.991 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (19m) 19s ago 27m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:53:24.991 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (6m) 4m ago 27m 81.8M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:53:24.991 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 19s ago 26m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (6m) 4m ago 31m 437M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (15m) 19s ago 33m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (4m) 19s ago 33m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (5m) 4m ago 32m 34.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (3m) 19s ago 32m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (19m) 19s ago 28m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (18m) 4m ago 28m 15.9M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (2m) 19s ago 31m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (2m) 19s ago 31m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (78s) 19s ago 30m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (24s) 19s ago 30m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (29m) 4m ago 29m 55.8M 3248M 17.2.0 e1d6a67b021e 67ce507923be 2024-09-17T01:53:24.992 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (29m) 4m ago 29m 52.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:53:24.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (29m) 4m ago 29m 53.5M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:53:24.993 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (28m) 4m ago 28m 54.7M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:53:24.993 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (6m) 4m ago 28m 65.4M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:53:24.993 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (26m) 19s ago 26m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:53:24.993 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (26m) 4m ago 26m 80.1M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:53:25.165 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:24 smithi105 ceph-mon[63427]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:53:25.165 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:25.170 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:24 smithi081 ceph-mon[77854]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:53:25.170 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:25.170 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:24 smithi081 ceph-mon[80183]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T01:53:25.170 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:25.407 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:53:25.408 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 9 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:53:25.409 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:53:25.750 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:53:25.750 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:53:25.750 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:53:25.750 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T01:53:25.750 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:53:25.751 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "4/8 daemons upgraded", 2024-09-17T01:53:25.751 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:53:25.751 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:53:25.751 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: Upgrade: osd.4 is safe to restart 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3743052153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:26.115 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: Upgrade: osd.4 is safe to restart 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.116 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:25 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3743052153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:53:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: Upgrade: osd.4 is safe to restart 2024-09-17T01:53:26.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:26.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:26.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:25 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3743052153' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:27.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[77854]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[77854]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[80183]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[80183]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:53:27.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:26 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:27.327 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:26 smithi105 ceph-mon[63427]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:27.327 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:26 smithi105 ceph-mon[63427]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:27.327 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:27.327 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:53:27.327 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:26 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:28.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:27 smithi081 ceph-mon[77854]: Upgrade: Updating osd.4 2024-09-17T01:53:28.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:27 smithi081 ceph-mon[77854]: Deploying daemon osd.4 on smithi105 2024-09-17T01:53:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:27 smithi081 ceph-mon[80183]: Upgrade: Updating osd.4 2024-09-17T01:53:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:27 smithi081 ceph-mon[80183]: Deploying daemon osd.4 on smithi105 2024-09-17T01:53:28.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:27 smithi105 ceph-mon[63427]: Upgrade: Updating osd.4 2024-09-17T01:53:28.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:27 smithi105 ceph-mon[63427]: Deploying daemon osd.4 on smithi105 2024-09-17T01:53:29.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:28 smithi105 ceph-mon[63427]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[77854]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:29.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:29.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:29.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[80183]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:29.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:29.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:29.564 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:29 smithi105 systemd[1]: Stopping Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:53:29.885 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:29 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[27934]: 2024-09-17T01:53:29.565+0000 7f749a18c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:53:29.885 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:29 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[27934]: 2024-09-17T01:53:29.565+0000 7f749a18c700 -1 osd.4 133 *** Got signal Terminated *** 2024-09-17T01:53:29.885 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:29 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[27934]: 2024-09-17T01:53:29.565+0000 7f749a18c700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:53:30.271 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:29 smithi105 ceph-mon[63427]: osd.4 marked itself down and dead 2024-09-17T01:53:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:29 smithi081 ceph-mon[77854]: osd.4 marked itself down and dead 2024-09-17T01:53:30.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:29 smithi081 ceph-mon[80183]: osd.4 marked itself down and dead 2024-09-17T01:53:30.591 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:30 smithi105 podman[72882]: 2024-09-17 01:53:30.273604793 +0000 UTC m=+0.809080416 container died 67ce507923be98d20709d4c69dee202b20078ce698b1628aa1a369b1856e70c4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, version=8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, ceph=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T01:53:30.888 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:30 smithi105 podman[72882]: 2024-09-17 01:53:30.68305877 +0000 UTC m=+1.218534378 container cleanup 67ce507923be98d20709d4c69dee202b20078ce698b1628aa1a369b1856e70c4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, release=754, vcs-type=git, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T01:53:30.889 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:30 smithi105 bash[72882]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4 2024-09-17T01:53:31.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:30 smithi105 ceph-mon[63427]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:31.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:30 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:53:31.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:30 smithi105 ceph-mon[63427]: osdmap e134: 8 total, 7 up, 8 in 2024-09-17T01:53:31.250 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:30 smithi105 podman[72897]: 2024-09-17 01:53:30.925980689 +0000 UTC m=+0.654615081 container remove 67ce507923be98d20709d4c69dee202b20078ce698b1628aa1a369b1856e70c4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, release=754, vcs-type=git, name=centos-stream, distribution-scope=public, io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8) 2024-09-17T01:53:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[77854]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:53:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[77854]: osdmap e134: 8 total, 7 up, 8 in 2024-09-17T01:53:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[80183]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:31.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:53:31.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:30 smithi081 ceph-mon[80183]: osdmap e134: 8 total, 7 up, 8 in 2024-09-17T01:53:31.577 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.252033076 +0000 UTC m=+0.105594596 container create 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T01:53:31.577 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.162819412 +0000 UTC m=+0.016380914 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:53:31.577 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.496645384 +0000 UTC m=+0.350206878 container init 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T01:53:31.577 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.499873228 +0000 UTC m=+0.353434707 container start 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T01:53:31.577 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.579692576 +0000 UTC m=+0.433254054 container attach 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T01:53:31.885 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:31 smithi105 podman[72985]: 2024-09-17 01:53:31.693818227 +0000 UTC m=+0.547379721 container died 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T01:53:32.322 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:31 smithi105 ceph-mon[63427]: osdmap e135: 8 total, 7 up, 8 in 2024-09-17T01:53:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:31 smithi081 ceph-mon[77854]: osdmap e135: 8 total, 7 up, 8 in 2024-09-17T01:53:32.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:31 smithi081 ceph-mon[80183]: osdmap e135: 8 total, 7 up, 8 in 2024-09-17T01:53:32.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 podman[73008]: 2024-09-17 01:53:32.324642746 +0000 UTC m=+0.625405675 container remove 7a2bcf768eff0b3fbe51585920f253470ab9021c9e3680d5c24790df4597cc5c (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T01:53:32.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T01:53:32.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 systemd[1]: Stopped Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:53:32.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service: Consumed 16.467s CPU time. 2024-09-17T01:53:32.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 systemd[1]: Starting Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:53:33.084 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:32 smithi105 ceph-mon[63427]: pgmap v177: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:53:33.084 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:32 smithi105 ceph-mon[63427]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T01:53:33.084 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 podman[73111]: 2024-09-17 01:53:32.654020798 +0000 UTC m=+0.016181545 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:53:33.084 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 podman[73111]: 2024-09-17 01:53:32.750985118 +0000 UTC m=+0.113145892 container create 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:53:33.084 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 podman[73111]: 2024-09-17 01:53:32.986805557 +0000 UTC m=+0.348966298 container init 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:53:33.085 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:32 smithi105 podman[73111]: 2024-09-17 01:53:32.990273339 +0000 UTC m=+0.352434076 container start 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:53:33.085 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:33 smithi105 podman[73111]: 2024-09-17 01:53:33.086728973 +0000 UTC m=+0.448889711 container attach 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T01:53:33.385 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:53:33.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:33 smithi105 bash[73111]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:53:33.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:53:33.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:33 smithi105 bash[73111]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:53:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:32 smithi081 ceph-mon[77854]: pgmap v177: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:53:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:32 smithi081 ceph-mon[77854]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T01:53:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:32 smithi081 ceph-mon[80183]: pgmap v177: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-09-17T01:53:33.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:32 smithi081 ceph-mon[80183]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T01:53:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:33] "GET /metrics HTTP/1.1" 200 38279 "" "Prometheus/2.43.0" 2024-09-17T01:53:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-mon[63427]: pgmap v178: 161 pgs: 16 active+undersized, 11 peering, 9 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-09-17T01:53:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:34.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:34 smithi081 ceph-mon[77854]: pgmap v178: 161 pgs: 16 active+undersized, 11 peering, 9 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-09-17T01:53:34.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:34 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:34.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:34 smithi081 ceph-mon[80183]: pgmap v178: 161 pgs: 16 active+undersized, 11 peering, 9 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 41/723 objects degraded (5.671%) 2024-09-17T01:53:34.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:34 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:53:34.779 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:53:35.112 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate[73126]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 bash[73111]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T01:53:35.113 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:34 smithi105 podman[73111]: 2024-09-17 01:53:34.817314885 +0000 UTC m=+2.179475629 container died 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2) 2024-09-17T01:53:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:35.773 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 podman[73254]: 2024-09-17 01:53:35.407523792 +0000 UTC m=+0.577767698 container remove 306587ad3f98625734674d39603a2ed1df569aae45c01a87076be6d7161bfc44 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T01:53:35.773 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 podman[73313]: 2024-09-17 01:53:35.528674112 +0000 UTC m=+0.016257561 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:53:35.773 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 podman[73313]: 2024-09-17 01:53:35.641775715 +0000 UTC m=+0.129359161 container create 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T01:53:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:36 smithi105 ceph-mon[63427]: pgmap v179: 161 pgs: 36 active+undersized, 11 peering, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 104/723 objects degraded (14.385%) 2024-09-17T01:53:36.385 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 podman[73313]: 2024-09-17 01:53:35.908620453 +0000 UTC m=+0.396203893 container init 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:53:36.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 podman[73313]: 2024-09-17 01:53:35.911912216 +0000 UTC m=+0.399495658 container start 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T01:53:36.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:35 smithi105 ceph-osd[73332]: -- 172.21.15.105:0/1987770410 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f93927fa40 con 0x55f93a046000 2024-09-17T01:53:36.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:36 smithi105 bash[73313]: 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 2024-09-17T01:53:36.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:36 smithi105 systemd[1]: Started Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:53:36.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:36 smithi081 ceph-mon[77854]: pgmap v179: 161 pgs: 36 active+undersized, 11 peering, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 104/723 objects degraded (14.385%) 2024-09-17T01:53:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:36 smithi081 ceph-mon[80183]: pgmap v179: 161 pgs: 36 active+undersized, 11 peering, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 104/723 objects degraded (14.385%) 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.200 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:37 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:38.065 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:53:37.813+0000 7f1fbf119640 -1 Falling back to public interface 2024-09-17T01:53:38.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:38 smithi105 ceph-mon[63427]: pgmap v180: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:38 smithi081 ceph-mon[77854]: pgmap v180: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:38.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:38 smithi081 ceph-mon[80183]: pgmap v180: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:39.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:39 smithi105 ceph-mon[63427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-17T01:53:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:39 smithi081 ceph-mon[77854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-17T01:53:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:39 smithi081 ceph-mon[80183]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-17T01:53:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:40 smithi105 ceph-mon[63427]: pgmap v181: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:40 smithi081 ceph-mon[77854]: pgmap v181: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:40.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:40 smithi081 ceph-mon[80183]: pgmap v181: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:41.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:53:41.326+0000 7f1fbf119640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T01:53:41.636 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:53:41.354+0000 7f1fbf119640 -1 osd.4 133 log_to_monitors true 2024-09-17T01:53:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:41.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:41 smithi105 ceph-mon[63427]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[80183]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 122/723 objects degraded (16.874%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:41.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:41.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:41.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:41 smithi081 ceph-mon[77854]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:53:42.518 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:53:42.176+0000 7f1fb6ec4640 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:53:42.818 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-mon[63427]: pgmap v182: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:42.818 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-mon[63427]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:53:42.818 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-mon[63427]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:42.819 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-mon[63427]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T01:53:42.819 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:42 smithi105 ceph-mon[63427]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:42.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[77854]: pgmap v182: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[77854]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[77854]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[77854]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[77854]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[80183]: pgmap v182: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 179 MiB used, 715 GiB / 715 GiB avail; 122/723 objects degraded (16.874%) 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[80183]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[80183]: from='osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[80183]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T01:53:42.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:42 smithi081 ceph-mon[80183]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:53:43.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:43 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:43.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:43 smithi105 ceph-mon[63427]: osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432] boot 2024-09-17T01:53:43.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:43 smithi105 ceph-mon[63427]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T01:53:43.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:53:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:43] "GET /metrics HTTP/1.1" 200 38279 "" "Prometheus/2.43.0" 2024-09-17T01:53:43.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[77854]: osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432] boot 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[77854]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[80183]: osd.4 [v2:172.21.15.105:6800/1163399432,v1:172.21.15.105:6801/1163399432] boot 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[80183]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T01:53:43.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:53:44.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:53:44.399+0000 7f469c54b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: pgmap v185: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:45.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: pgmap v185: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: pgmap v185: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:45.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:45 smithi105 ceph-mon[63427]: pgmap v187: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:45 smithi105 ceph-mon[63427]: pgmap v188: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:45 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:45 smithi105 ceph-mon[63427]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T01:53:46.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:45 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:46.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[77854]: pgmap v187: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[77854]: pgmap v188: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[77854]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[80183]: pgmap v187: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[80183]: pgmap v188: 161 pgs: 44 active+undersized, 24 active+undersized+degraded, 93 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 122/723 objects degraded (16.874%) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[80183]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T01:53:46.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:45 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:46 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:46 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:46 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:53:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:47 smithi105 ceph-mon[63427]: pgmap v189: 161 pgs: 23 active+undersized, 14 active+undersized+degraded, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-09-17T01:53:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:47 smithi081 ceph-mon[77854]: pgmap v189: 161 pgs: 23 active+undersized, 14 active+undersized+degraded, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-09-17T01:53:48.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:47 smithi081 ceph-mon[80183]: pgmap v189: 161 pgs: 23 active+undersized, 14 active+undersized+degraded, 124 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 57/723 objects degraded (7.884%) 2024-09-17T01:53:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:48 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-09-17T01:53:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:48 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-09-17T01:53:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:48 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 14 pgs degraded) 2024-09-17T01:53:50.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:49 smithi081 ceph-mon[77854]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-09-17T01:53:50.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:49 smithi081 ceph-mon[80183]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-09-17T01:53:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:49 smithi105 ceph-mon[63427]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-09-17T01:53:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:51 smithi081 ceph-mon[77854]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 760 B/s rd, 0 op/s 2024-09-17T01:53:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:51 smithi081 ceph-mon[80183]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 760 B/s rd, 0 op/s 2024-09-17T01:53:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:51 smithi105 ceph-mon[63427]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 760 B/s rd, 0 op/s 2024-09-17T01:53:53.929 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:53:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:53:53] "GET /metrics HTTP/1.1" 200 38149 "" "Prometheus/2.43.0" 2024-09-17T01:53:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:53 smithi081 ceph-mon[77854]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:54.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:53 smithi081 ceph-mon[80183]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:54.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:54.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:53 smithi105 ceph-mon[63427]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:53:54.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:56.113 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:53:56.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:55 smithi105 ceph-mon[63427]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:53:56.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:55 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:55 smithi081 ceph-mon[80183]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:53:56.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:55 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:56.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:55 smithi081 ceph-mon[77854]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:53:56.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:55 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (19m) 50s ago 27m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (6m) 16s ago 27m 77.2M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 50s ago 26m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (6m) 16s ago 32m 441M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (16m) 50s ago 33m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (4m) 50s ago 34m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (5m) 16s ago 32m 44.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:53:56.757 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (4m) 50s ago 32m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (20m) 50s ago 28m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (19m) 16s ago 28m 15.3M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (3m) 50s ago 31m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (2m) 50s ago 31m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (110s) 50s ago 31m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (55s) 50s ago 30m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (21s) 16s ago 30m 11.7M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (30m) 16s ago 30m 56.4M 3248M 17.2.0 e1d6a67b021e 94807db23f28 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (29m) 16s ago 29m 56.8M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (29m) 16s ago 29m 58.3M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (6m) 16s ago 28m 67.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:53:56.758 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (27m) 50s ago 27m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:53:56.759 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (27m) 16s ago 27m 81.2M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:53:57.156 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 10 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:53:57.157 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "5/8 daemons upgraded", 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:53:57.486 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:53:58.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:57 smithi105 ceph-mon[63427]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:57 smithi105 ceph-mon[63427]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:58.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:57 smithi105 ceph-mon[63427]: from='client.64444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:57 smithi105 ceph-mon[63427]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:57 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3640437703' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:58.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[80183]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[80183]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[80183]: from='client.64444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[80183]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3640437703' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[77854]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[77854]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[77854]: from='client.64444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[77854]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:58.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:57 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3640437703' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:53:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:58 smithi105 ceph-mon[63427]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:59.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[77854]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:59.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:53:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[80183]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:53:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:53:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:00.066 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:59 smithi105 ceph-mon[63427]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:00.066 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:53:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:59 smithi081 ceph-mon[77854]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:00.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:53:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:59 smithi081 ceph-mon[80183]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:53:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:00 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:00 smithi105 ceph-mon[63427]: Upgrade: osd.5 is safe to restart 2024-09-17T01:54:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:00 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:00 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:54:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:00 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:01.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:01.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[77854]: Upgrade: osd.5 is safe to restart 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[80183]: Upgrade: osd.5 is safe to restart 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:54:01.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:00 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:02.202 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:01 smithi105 ceph-mon[63427]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:02.202 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:01 smithi105 ceph-mon[63427]: Upgrade: Updating osd.5 2024-09-17T01:54:02.202 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:01 smithi105 ceph-mon[63427]: Deploying daemon osd.5 on smithi105 2024-09-17T01:54:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[77854]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[77854]: Upgrade: Updating osd.5 2024-09-17T01:54:02.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[77854]: Deploying daemon osd.5 on smithi105 2024-09-17T01:54:02.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[80183]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:02.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[80183]: Upgrade: Updating osd.5 2024-09-17T01:54:02.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:01 smithi081 ceph-mon[80183]: Deploying daemon osd.5 on smithi105 2024-09-17T01:54:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:03] "GET /metrics HTTP/1.1" 200 38289 "" "Prometheus/2.43.0" 2024-09-17T01:54:03.955 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:03 smithi105 systemd[1]: Stopping Ceph osd.5 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:54:04.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:03 smithi105 ceph-mon[63427]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:04.386 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[30862]: 2024-09-17T01:54:03.983+0000 7fb201e39700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:54:04.386 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[30862]: 2024-09-17T01:54:03.983+0000 7fb201e39700 -1 osd.5 138 *** Got signal Terminated *** 2024-09-17T01:54:04.386 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[30862]: 2024-09-17T01:54:03.983+0000 7fb201e39700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:54:04.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:03 smithi081 ceph-mon[77854]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:04.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:03 smithi081 ceph-mon[80183]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:05.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:04 smithi105 ceph-mon[63427]: osd.5 marked itself down and dead 2024-09-17T01:54:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:04 smithi081 ceph-mon[77854]: osd.5 marked itself down and dead 2024-09-17T01:54:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:04 smithi081 ceph-mon[80183]: osd.5 marked itself down and dead 2024-09-17T01:54:05.749 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:05 smithi105 podman[77913]: 2024-09-17 01:54:05.435782452 +0000 UTC m=+1.553647913 container died 94807db23f281b7cb6384c441db7ce010ce54c9a57c098e26df31a4c68f3e0df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-type=git, GIT_CLEAN=True) 2024-09-17T01:54:06.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:05 smithi105 ceph-mon[63427]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:06.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:05 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:06.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:05 smithi105 ceph-mon[63427]: osdmap e139: 8 total, 7 up, 8 in 2024-09-17T01:54:06.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:05 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:06.100 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:05 smithi105 podman[77913]: 2024-09-17 01:54:05.858681932 +0000 UTC m=+1.976547396 container cleanup 94807db23f281b7cb6384c441db7ce010ce54c9a57c098e26df31a4c68f3e0df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git, distribution-scope=public, release=754, RELEASE=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.buildah.version=1.19.8, version=8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T01:54:06.100 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:05 smithi105 bash[77913]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5 2024-09-17T01:54:06.385 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[77928]: 2024-09-17 01:54:06.101795567 +0000 UTC m=+0.664216980 container remove 94807db23f281b7cb6384c441db7ce010ce54c9a57c098e26df31a4c68f3e0df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, io.openshift.expose-services=) 2024-09-17T01:54:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[77854]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[77854]: osdmap e139: 8 total, 7 up, 8 in 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[80183]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[80183]: osdmap e139: 8 total, 7 up, 8 in 2024-09-17T01:54:06.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:05 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:06.795 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.430244601 +0000 UTC m=+0.016513188 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:06.795 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.552845023 +0000 UTC m=+0.139113596 container create b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:54:07.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:06 smithi105 ceph-mon[63427]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T01:54:07.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.797293359 +0000 UTC m=+0.383561942 container init b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:54:07.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.800660623 +0000 UTC m=+0.386929206 container start b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-c2ddf97) 2024-09-17T01:54:07.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.897087978 +0000 UTC m=+0.483356565 container attach b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:54:07.137 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:06 smithi105 podman[78016]: 2024-09-17 01:54:06.992569275 +0000 UTC m=+0.578837859 container died b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:54:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:06 smithi081 ceph-mon[77854]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T01:54:07.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:06 smithi081 ceph-mon[80183]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T01:54:07.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 podman[78039]: 2024-09-17 01:54:07.550981235 +0000 UTC m=+0.552538517 container remove b86a917d3fcba378de6668ff37d9297ef6c5c77b1b5edab07c47406132f4b8ad (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:54:07.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T01:54:07.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 systemd[1]: Stopped Ceph osd.5 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:54:07.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.5.service: Consumed 15.669s CPU time. 2024-09-17T01:54:07.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 systemd[1]: Starting Ceph osd.5 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:54:08.269 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:07 smithi105 ceph-mon[63427]: pgmap v201: 161 pgs: 13 peering, 16 stale+active+clean, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:54:08.269 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:08 smithi105 ceph-mon[63427]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-17T01:54:08.269 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:07 smithi105 podman[78136]: 2024-09-17 01:54:07.88720874 +0000 UTC m=+0.017514836 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:08.269 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 podman[78136]: 2024-09-17 01:54:08.143621708 +0000 UTC m=+0.273927796 container create fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T01:54:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:07 smithi081 ceph-mon[77854]: pgmap v201: 161 pgs: 13 peering, 16 stale+active+clean, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:54:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:07 smithi081 ceph-mon[77854]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-17T01:54:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:07 smithi081 ceph-mon[80183]: pgmap v201: 161 pgs: 13 peering, 16 stale+active+clean, 132 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:54:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:07 smithi081 ceph-mon[80183]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-17T01:54:08.524 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 podman[78136]: 2024-09-17 01:54:08.379762767 +0000 UTC m=+0.510068868 container init fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 podman[78136]: 2024-09-17 01:54:08.38365254 +0000 UTC m=+0.513958628 container start fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=) 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 podman[78136]: 2024-09-17 01:54:08.454689008 +0000 UTC m=+0.584995109 container attach fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 bash[78136]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:08.525 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:08 smithi105 bash[78136]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-mon[63427]: pgmap v202: 161 pgs: 19 active+undersized, 13 peering, 5 stale+active+clean, 11 active+undersized+degraded, 113 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 46/723 objects degraded (6.362%) 2024-09-17T01:54:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 46/723 objects degraded (6.362%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:09 smithi081 ceph-mon[77854]: pgmap v202: 161 pgs: 19 active+undersized, 13 peering, 5 stale+active+clean, 11 active+undersized+degraded, 113 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 46/723 objects degraded (6.362%) 2024-09-17T01:54:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:09 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 46/723 objects degraded (6.362%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:09.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:09 smithi081 ceph-mon[80183]: pgmap v202: 161 pgs: 19 active+undersized, 13 peering, 5 stale+active+clean, 11 active+undersized+degraded, 113 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 46/723 objects degraded (6.362%) 2024-09-17T01:54:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:09 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 46/723 objects degraded (6.362%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:54:09.862 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate[78150]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T01:54:10.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 bash[78136]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T01:54:10.137 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:09 smithi105 podman[78136]: 2024-09-17 01:54:09.891939196 +0000 UTC m=+2.022245292 container died fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-c2ddf97, GIT_CLEAN=True) 2024-09-17T01:54:10.850 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 podman[78277]: 2024-09-17 01:54:10.516821154 +0000 UTC m=+0.618913953 container remove fc5ee0577e22375696f78142b96be09e41132702a1742df81329160f8fa815fe (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True) 2024-09-17T01:54:10.850 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 podman[78337]: 2024-09-17 01:54:10.625390229 +0000 UTC m=+0.016053244 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:10.850 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 podman[78337]: 2024-09-17 01:54:10.734384983 +0000 UTC m=+0.125048031 container create 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T01:54:11.344 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 podman[78337]: 2024-09-17 01:54:10.953355146 +0000 UTC m=+0.344018151 container init 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:54:11.345 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 podman[78337]: 2024-09-17 01:54:10.956688925 +0000 UTC m=+0.347351933 container start 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:54:11.345 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:10 smithi105 ceph-osd[78355]: -- 172.21.15.105:0/1031612601 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b6656a3a40 con 0x55b66646a400 2024-09-17T01:54:11.345 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:11 smithi105 bash[78337]: 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 2024-09-17T01:54:11.345 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:11 smithi105 systemd[1]: Started Ceph osd.5 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:54:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:11 smithi105 ceph-mon[63427]: pgmap v203: 161 pgs: 27 active+undersized, 13 peering, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-09-17T01:54:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:11 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:11.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:11 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[77854]: pgmap v203: 161 pgs: 27 active+undersized, 13 peering, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-09-17T01:54:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:11.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:11.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[80183]: pgmap v203: 161 pgs: 27 active+undersized, 13 peering, 19 active+undersized+degraded, 102 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 69/723 objects degraded (9.544%) 2024-09-17T01:54:11.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:11.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:11 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:12.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:12 smithi081 ceph-mon[77854]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-17T01:54:12.680 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:12 smithi081 ceph-mon[80183]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-17T01:54:12.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:12 smithi105 ceph-mon[63427]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-17T01:54:12.885 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:12 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:54:12.588+0000 7fd3e39a1640 -1 Falling back to public interface 2024-09-17T01:54:13.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:13 smithi081 ceph-mon[77854]: pgmap v204: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:13.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:13 smithi081 ceph-mon[80183]: pgmap v204: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:13.681 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:13] "GET /metrics HTTP/1.1" 200 38289 "" "Prometheus/2.43.0" 2024-09-17T01:54:13.727 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:13 smithi105 ceph-mon[63427]: pgmap v204: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:15.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:16.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:15 smithi105 ceph-mon[63427]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:16.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:15 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:16.105 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:15 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.105 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:15 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[77854]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[80183]: pgmap v205: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:15 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:16.385 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:54:16.105+0000 7fd3e39a1640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T01:54:16.385 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:54:16.138+0000 7fd3e39a1640 -1 osd.5 138 log_to_monitors true 2024-09-17T01:54:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-mon[63427]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-mon[63427]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:16 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[77854]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[77854]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 82/723 objects degraded (11.342%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[80183]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[80183]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:54:17.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:16 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:17.543 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:54:17.179+0000 7fd3db74c640 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:54:18.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-mon[63427]: pgmap v206: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:18.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-mon[63427]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:54:18.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-mon[63427]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T01:54:18.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-mon[63427]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:18.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:17 smithi105 ceph-mon[63427]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:18.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[77854]: pgmap v206: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:18.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[77854]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[77854]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[77854]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[77854]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[80183]: pgmap v206: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 195 MiB used, 715 GiB / 715 GiB avail; 82/723 objects degraded (11.342%) 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[80183]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[80183]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[80183]: from='osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:18.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:17 smithi081 ceph-mon[80183]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:18 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:18 smithi105 ceph-mon[63427]: osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509] boot 2024-09-17T01:54:19.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:18 smithi105 ceph-mon[63427]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T01:54:19.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:18 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:54:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[77854]: osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509] boot 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[77854]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[80183]: osd.5 [v2:172.21.15.105:6808/4254239509,v1:172.21.15.105:6809/4254239509] boot 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[80183]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T01:54:19.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:18 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:54:19.832 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:54:19.476+0000 7f469c54b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:20.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: pgmap v209: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:19 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:20.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: pgmap v209: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:20.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:20.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: pgmap v209: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:20.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:19 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:21.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:20 smithi105 ceph-mon[63427]: pgmap v211: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:20 smithi105 ceph-mon[63427]: pgmap v212: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:20 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:21.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:20 smithi105 ceph-mon[63427]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:21.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[77854]: pgmap v211: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[77854]: pgmap v212: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[77854]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:21.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[80183]: pgmap v211: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[80183]: pgmap v212: 161 pgs: 37 active+undersized, 22 active+undersized+degraded, 102 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s; 82/723 objects degraded (11.342%) 2024-09-17T01:54:21.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:21.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:20 smithi081 ceph-mon[80183]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:22.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:21 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:21 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:22.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:21 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:22 smithi081 ceph-mon[77854]: pgmap v213: 161 pgs: 26 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-09-17T01:54:23.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:22 smithi081 ceph-mon[80183]: pgmap v213: 161 pgs: 26 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-09-17T01:54:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:22 smithi105 ceph-mon[63427]: pgmap v213: 161 pgs: 26 active+undersized, 14 active+undersized+degraded, 121 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 54/723 objects degraded (7.469%) 2024-09-17T01:54:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:23] "GET /metrics HTTP/1.1" 200 38290 "" "Prometheus/2.43.0" 2024-09-17T01:54:24.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:23 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-09-17T01:54:24.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:23 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-09-17T01:54:24.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:23 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-09-17T01:54:25.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:24 smithi105 ceph-mon[63427]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:25.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:24 smithi081 ceph-mon[77854]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:25.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:24 smithi081 ceph-mon[80183]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:26.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:25 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:26.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:25 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:26.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:25 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:27.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:26 smithi105 ceph-mon[63427]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:27.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:26 smithi081 ceph-mon[77854]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:27.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:26 smithi081 ceph-mon[80183]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:27.845 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:54:28.491 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (20m) 82s ago 28m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (7m) 12s ago 28m 82.0M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 82s ago 27m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (7m) 12s ago 32m 441M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (16m) 82s ago 34m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (5m) 82s ago 34m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (6m) 12s ago 33m 46.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (5m) 82s ago 33m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (20m) 82s ago 29m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:54:28.492 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (20m) 12s ago 29m 15.3M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (3m) 82s ago 32m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (3m) 82s ago 32m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (2m) 82s ago 31m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (87s) 82s ago 31m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (52s) 12s ago 31m 45.0M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (17s) 12s ago 30m 11.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (30m) 12s ago 30m 58.7M 3248M 17.2.0 e1d6a67b021e 99ea20a2fb78 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (29m) 12s ago 29m 58.8M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (7m) 12s ago 29m 67.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (27m) 82s ago 27m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:54:28.493 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (27m) 12s ago 27m 81.4M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:54:28.887 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:54:28.888 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 11 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:54:28.889 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:54:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-17T01:54:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:29.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1588097165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1588097165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:29.223 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:54:29.223 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "6/8 daemons upgraded", 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:54:29.224 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1588097165' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:30.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:29 smithi105 ceph-mon[63427]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:30.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:29 smithi105 ceph-mon[63427]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:29 smithi081 ceph-mon[77854]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:30.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:29 smithi081 ceph-mon[77854]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:30.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:29 smithi081 ceph-mon[80183]: from='client.54510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:30.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:29 smithi081 ceph-mon[80183]: from='client.54516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:30 smithi105 ceph-mon[63427]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:31.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:30 smithi105 ceph-mon[63427]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:54:31.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:30 smithi081 ceph-mon[80183]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:31.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:30 smithi081 ceph-mon[80183]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:54:31.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:30 smithi081 ceph-mon[77854]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:54:31.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:30 smithi081 ceph-mon[77854]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T01:54:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:32 smithi105 ceph-mon[63427]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:33.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:32 smithi081 ceph-mon[80183]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:33.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:32 smithi081 ceph-mon[77854]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:54:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:33] "GET /metrics HTTP/1.1" 200 38300 "" "Prometheus/2.43.0" 2024-09-17T01:54:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:34 smithi105 ceph-mon[63427]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:34 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:34 smithi081 ceph-mon[77854]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:35.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:34 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:34 smithi081 ceph-mon[80183]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-09-17T01:54:35.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:34 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: Upgrade: osd.6 is safe to restart 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:54:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:35 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:36.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: Upgrade: osd.6 is safe to restart 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: Upgrade: osd.6 is safe to restart 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:54:36.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:35 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:37.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:36 smithi105 ceph-mon[63427]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:37.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:36 smithi105 ceph-mon[63427]: Upgrade: Updating osd.6 2024-09-17T01:54:37.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:36 smithi105 ceph-mon[63427]: Deploying daemon osd.6 on smithi105 2024-09-17T01:54:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[77854]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[77854]: Upgrade: Updating osd.6 2024-09-17T01:54:37.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[77854]: Deploying daemon osd.6 on smithi105 2024-09-17T01:54:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[80183]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[80183]: Upgrade: Updating osd.6 2024-09-17T01:54:37.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:36 smithi081 ceph-mon[80183]: Deploying daemon osd.6 on smithi105 2024-09-17T01:54:38.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:38 smithi105 systemd[1]: Stopping Ceph osd.6 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:54:39.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:38 smithi105 ceph-mon[63427]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:39.255 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[33740]: 2024-09-17T01:54:39.071+0000 7fd30196b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:54:39.255 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[33740]: 2024-09-17T01:54:39.071+0000 7fd30196b700 -1 osd.6 143 *** Got signal Terminated *** 2024-09-17T01:54:39.256 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[33740]: 2024-09-17T01:54:39.071+0000 7fd30196b700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:54:39.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:38 smithi081 ceph-mon[80183]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:39.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:38 smithi081 ceph-mon[77854]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:54:40.321 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:39 smithi105 ceph-mon[63427]: osd.6 marked itself down and dead 2024-09-17T01:54:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:39 smithi081 ceph-mon[77854]: osd.6 marked itself down and dead 2024-09-17T01:54:40.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:39 smithi081 ceph-mon[80183]: osd.6 marked itself down and dead 2024-09-17T01:54:40.628 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:40 smithi105 podman[82384]: 2024-09-17 01:54:40.323131992 +0000 UTC m=+1.356718560 container died 99ea20a2fb78e2d5dc8c8385930fe1c6bad5df5e1a9fda823a944b535bbaf5ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, release=754, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream, vendor=Red Hat, Inc., vcs-type=git) 2024-09-17T01:54:40.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:40 smithi105 podman[82384]: 2024-09-17 01:54:40.713166621 +0000 UTC m=+1.746753179 container cleanup 99ea20a2fb78e2d5dc8c8385930fe1c6bad5df5e1a9fda823a944b535bbaf5ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , RELEASE=HEAD, version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, release=754, architecture=x86_64, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-09-17T01:54:40.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:40 smithi105 bash[82384]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6 2024-09-17T01:54:41.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:40 smithi105 ceph-mon[63427]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:41.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:40 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:41.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:40 smithi105 ceph-mon[63427]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T01:54:41.293 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:40 smithi105 podman[82412]: 2024-09-17 01:54:40.964702167 +0000 UTC m=+0.636518861 container remove 99ea20a2fb78e2d5dc8c8385930fe1c6bad5df5e1a9fda823a944b535bbaf5ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, release=754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T01:54:41.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[77854]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:41.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:41.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[77854]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T01:54:41.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[80183]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:54:41.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:54:41.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:40 smithi081 ceph-mon[80183]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T01:54:41.549 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.198622949 +0000 UTC m=+0.016151569 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:41.549 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.315705869 +0000 UTC m=+0.133234474 container create 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T01:54:41.549 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.551340628 +0000 UTC m=+0.368869238 container init 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T01:54:41.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.554794564 +0000 UTC m=+0.372323169 container start 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T01:54:41.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.653564272 +0000 UTC m=+0.471092886 container attach 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:54:41.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:41 smithi105 podman[82501]: 2024-09-17 01:54:41.745685453 +0000 UTC m=+0.563214059 container died 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:54:42.363 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:41 smithi105 ceph-mon[63427]: osdmap e145: 8 total, 7 up, 8 in 2024-09-17T01:54:42.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:41 smithi081 ceph-mon[77854]: osdmap e145: 8 total, 7 up, 8 in 2024-09-17T01:54:42.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:41 smithi081 ceph-mon[80183]: osdmap e145: 8 total, 7 up, 8 in 2024-09-17T01:54:42.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 podman[82524]: 2024-09-17 01:54:42.36556574 +0000 UTC m=+0.610131168 container remove 155f6577b55eb35f4a600ab3ed39bbeab02814e334c4f6e805714358c7112e10 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-17T01:54:42.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T01:54:42.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 systemd[1]: Stopped Ceph osd.6 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:54:42.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.6.service: Consumed 13.927s CPU time. 2024-09-17T01:54:42.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 systemd[1]: Starting Ceph osd.6 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:54:43.083 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:42 smithi105 ceph-mon[63427]: pgmap v225: 161 pgs: 4 peering, 13 stale+active+clean, 144 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:54:43.084 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 podman[82624]: 2024-09-17 01:54:42.703563017 +0000 UTC m=+0.016341683 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:43.084 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:42 smithi105 podman[82624]: 2024-09-17 01:54:42.816602496 +0000 UTC m=+0.129381149 container create d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97) 2024-09-17T01:54:43.084 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 podman[82624]: 2024-09-17 01:54:43.085931689 +0000 UTC m=+0.398710344 container init d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 podman[82624]: 2024-09-17 01:54:43.089311683 +0000 UTC m=+0.402090338 container start d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 podman[82624]: 2024-09-17 01:54:43.185947926 +0000 UTC m=+0.498726582 container attach d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 bash[82624]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:43.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 bash[82624]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:54:43.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:42 smithi081 ceph-mon[77854]: pgmap v225: 161 pgs: 4 peering, 13 stale+active+clean, 144 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:54:43.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:42 smithi081 ceph-mon[80183]: pgmap v225: 161 pgs: 4 peering, 13 stale+active+clean, 144 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:54:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:43] "GET /metrics HTTP/1.1" 200 38300 "" "Prometheus/2.43.0" 2024-09-17T01:54:44.283 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:54:44.283 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 bash[82624]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:54:44.283 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:54:44.284 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:43 smithi105 bash[82624]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:54:44.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:54:44.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 bash[82624]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 bash[82624]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 bash[82624]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 bash[82624]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate[82638]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 bash[82624]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T01:54:44.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:44 smithi105 podman[82624]: 2024-09-17 01:54:44.314584313 +0000 UTC m=+1.627362978 container died d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T01:54:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-mon[63427]: pgmap v226: 161 pgs: 24 active+undersized, 4 peering, 1 stale+active+clean, 15 active+undersized+degraded, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 58/723 objects degraded (8.022%) 2024-09-17T01:54:45.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:45.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:44 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:45.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[77854]: pgmap v226: 161 pgs: 24 active+undersized, 4 peering, 1 stale+active+clean, 15 active+undersized+degraded, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 58/723 objects degraded (8.022%) 2024-09-17T01:54:45.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:45.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:45.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[80183]: pgmap v226: 161 pgs: 24 active+undersized, 4 peering, 1 stale+active+clean, 15 active+undersized+degraded, 117 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 58/723 objects degraded (8.022%) 2024-09-17T01:54:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:45.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:44 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:45.518 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 podman[82778]: 2024-09-17 01:54:45.173693212 +0000 UTC m=+0.852873378 container remove d611903ecb4ab22e3c0ac209459e1e75529b87c65e46fa6a8bd259a1f0900832 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-17T01:54:45.518 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 podman[82837]: 2024-09-17 01:54:45.29142922 +0000 UTC m=+0.016390058 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:54:45.518 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 podman[82837]: 2024-09-17 01:54:45.408117358 +0000 UTC m=+0.133078178 container create 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T01:54:45.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 podman[82837]: 2024-09-17 01:54:45.627325361 +0000 UTC m=+0.352286192 container init 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:54:45.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 podman[82837]: 2024-09-17 01:54:45.630509306 +0000 UTC m=+0.355470133 container start 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:54:45.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 ceph-osd[82856]: -- 172.21.15.105:0/2565270011 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a27b937a40 con 0x55a27c6fe400 2024-09-17T01:54:45.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 bash[82837]: 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 2024-09-17T01:54:45.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:45 smithi105 systemd[1]: Started Ceph osd.6 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:54:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:46 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:46 smithi105 ceph-mon[63427]: pgmap v227: 161 pgs: 31 active+undersized, 4 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:54:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[77854]: pgmap v227: 161 pgs: 31 active+undersized, 4 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:54:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[80183]: pgmap v227: 161 pgs: 31 active+undersized, 4 peering, 18 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:54:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:47.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:54:47.271+0000 7fd239423640 -1 Falling back to public interface 2024-09-17T01:54:48.880 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:48 smithi105 ceph-mon[63427]: pgmap v228: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:48 smithi081 ceph-mon[77854]: pgmap v228: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:49.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:48 smithi081 ceph-mon[80183]: pgmap v228: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:50.844 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:54:50.552+0000 7fd239423640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T01:54:50.845 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:54:50.574+0000 7fd239423640 -1 osd.6 143 log_to_monitors true 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: pgmap v229: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.122 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:50 smithi105 ceph-mon[63427]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: pgmap v229: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[77854]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: pgmap v229: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:50 smithi081 ceph-mon[80183]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:54:51.850 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:54:51.491+0000 7fd2311ce640 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:54:52.112 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:52.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-mon[63427]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:54:52.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-mon[63427]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:52.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-mon[63427]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T01:54:52.113 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:51 smithi105 ceph-mon[63427]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[77854]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:54:52.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[77854]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[77854]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[77854]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[80183]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[80183]: from='osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[80183]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T01:54:52.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:51 smithi081 ceph-mon[80183]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:54:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:52 smithi105 ceph-mon[63427]: pgmap v230: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:52 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:52 smithi105 ceph-mon[63427]: osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176] boot 2024-09-17T01:54:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:52 smithi105 ceph-mon[63427]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T01:54:53.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:52 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:54:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[77854]: pgmap v230: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[77854]: osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176] boot 2024-09-17T01:54:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[77854]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[80183]: pgmap v230: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 211 MiB used, 715 GiB / 715 GiB avail; 197 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[80183]: osd.6 [v2:172.21.15.105:6816/1385727176,v1:172.21.15.105:6817/1385727176] boot 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[80183]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T01:54:53.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:52 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:54:53.765 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:54:53] "GET /metrics HTTP/1.1" 200 38299 "" "Prometheus/2.43.0" 2024-09-17T01:54:54.180 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:54:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:54:53.767+0000 7f469c54b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: pgmap v233: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: pgmap v235: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: pgmap v236: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:54 smithi105 ceph-mon[63427]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:54.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: pgmap v233: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T01:54:54.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: pgmap v235: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: pgmap v236: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[77854]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:54.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: pgmap v233: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T01:54:54.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.934 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: pgmap v235: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: pgmap v236: 161 pgs: 34 active+undersized, 19 active+undersized+degraded, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s; 85/723 objects degraded (11.757%) 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:54:54.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:54 smithi081 ceph-mon[80183]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T01:54:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:56 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:56 smithi105 ceph-mon[63427]: pgmap v237: 161 pgs: 28 active+undersized, 16 active+undersized+degraded, 117 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 69/723 objects degraded (9.544%) 2024-09-17T01:54:57.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:56 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:57.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:57.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[77854]: pgmap v237: 161 pgs: 28 active+undersized, 16 active+undersized+degraded, 117 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 69/723 objects degraded (9.544%) 2024-09-17T01:54:57.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:54:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[80183]: pgmap v237: 161 pgs: 28 active+undersized, 16 active+undersized+degraded, 117 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 69/723 objects degraded (9.544%) 2024-09-17T01:54:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:56 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:54:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:57 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-09-17T01:54:58.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:57 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-09-17T01:54:58.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:57 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/723 objects degraded (9.544%), 16 pgs degraded) 2024-09-17T01:54:59.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:58 smithi105 ceph-mon[63427]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-09-17T01:54:59.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:58 smithi081 ceph-mon[77854]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-09-17T01:54:59.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:58 smithi081 ceph-mon[80183]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-09-17T01:54:59.579 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:54:59.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:59.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:54:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:54:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:54:59.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:54:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:00.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:00.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:54:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (20m) 114s ago 28m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (7m) 10s ago 28m 82.1M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 114s ago 27m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (7m) 10s ago 33m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (17m) 114s ago 34m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (5m) 114s ago 35m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (6m) 10s ago 33m 47.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:55:00.231 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (5m) 114s ago 33m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (21m) 114s ago 29m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (20m) 10s ago 29m 15.7M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (4m) 114s ago 33m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (3m) 114s ago 32m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (2m) 114s ago 32m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (119s) 114s ago 31m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (84s) 10s ago 31m 45.7M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (49s) 10s ago 31m 43.8M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (14s) 10s ago 30m 12.1M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (30m) 10s ago 30m 60.2M 3248M 17.2.0 e1d6a67b021e 2fb3cc0892e6 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (8m) 10s ago 29m 67.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (28m) 114s ago 28m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:55:00.232 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (28m) 10s ago 28m 81.5M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:55:00.626 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 12 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:55:00.627 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:55:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[77854]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[77854]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-09-17T01:55:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[77854]: from='client.54540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:00.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3324774659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:00.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[80183]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:00.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[80183]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-09-17T01:55:00.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[80183]: from='client.54540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:00.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:00 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3324774659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:00.970 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:55:00.970 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:55:00.970 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:55:00.970 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T01:55:00.970 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:55:00.971 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "7/8 daemons upgraded", 2024-09-17T01:55:00.971 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:55:00.971 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:55:00.971 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:55:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:00 smithi105 ceph-mon[63427]: from='client.64501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:00 smithi105 ceph-mon[63427]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 652 B/s rd, 0 op/s 2024-09-17T01:55:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:00 smithi105 ceph-mon[63427]: from='client.54540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:00 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3324774659' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:01 smithi105 ceph-mon[63427]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:02.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:01 smithi105 ceph-mon[63427]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:01 smithi081 ceph-mon[77854]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:01 smithi081 ceph-mon[77854]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:02.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:01 smithi081 ceph-mon[80183]: from='client.54546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:02.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:01 smithi081 ceph-mon[80183]: from='client.64525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:03.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:02 smithi105 ceph-mon[63427]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 501 B/s rd, 0 op/s 2024-09-17T01:55:03.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:02 smithi081 ceph-mon[77854]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 501 B/s rd, 0 op/s 2024-09-17T01:55:03.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:02 smithi081 ceph-mon[80183]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 501 B/s rd, 0 op/s 2024-09-17T01:55:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:03] "GET /metrics HTTP/1.1" 200 38315 "" "Prometheus/2.43.0" 2024-09-17T01:55:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:04 smithi105 ceph-mon[63427]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-17T01:55:05.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:04 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:04 smithi081 ceph-mon[77854]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-17T01:55:05.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:04 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:05.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:04 smithi081 ceph-mon[80183]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-09-17T01:55:05.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:04 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:06 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:06 smithi105 ceph-mon[63427]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:55:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:06 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:07.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:06 smithi081 ceph-mon[80183]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:55:07.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:06 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:07.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:06 smithi081 ceph-mon[77854]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:55:09.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:08 smithi105 ceph-mon[63427]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:09.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:08 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:08 smithi081 ceph-mon[77854]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:09.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:08 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:08 smithi081 ceph-mon[80183]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:09.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:08 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:09 smithi105 ceph-mon[63427]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:09 smithi105 ceph-mon[63427]: Upgrade: osd.7 is safe to restart 2024-09-17T01:55:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:09 smithi081 ceph-mon[77854]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:09 smithi081 ceph-mon[77854]: Upgrade: osd.7 is safe to restart 2024-09-17T01:55:10.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:09 smithi081 ceph-mon[80183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:55:10.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:09 smithi081 ceph-mon[80183]: Upgrade: osd.7 is safe to restart 2024-09-17T01:55:11.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:10 smithi105 ceph-mon[63427]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:11.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:11.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:55:11.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[77854]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:55:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[80183]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:55:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:12.131 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:11 smithi105 ceph-mon[63427]: Upgrade: Updating osd.7 2024-09-17T01:55:12.132 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:11 smithi105 ceph-mon[63427]: Deploying daemon osd.7 on smithi105 2024-09-17T01:55:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:11 smithi081 ceph-mon[77854]: Upgrade: Updating osd.7 2024-09-17T01:55:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:11 smithi081 ceph-mon[77854]: Deploying daemon osd.7 on smithi105 2024-09-17T01:55:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:11 smithi081 ceph-mon[80183]: Upgrade: Updating osd.7 2024-09-17T01:55:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:11 smithi081 ceph-mon[80183]: Deploying daemon osd.7 on smithi105 2024-09-17T01:55:13.134 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:12 smithi105 ceph-mon[63427]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:13.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:13 smithi105 systemd[1]: Stopping Ceph osd.7 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:55:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:12 smithi081 ceph-mon[77854]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:13.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:12 smithi081 ceph-mon[80183]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:13.885 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[36645]: 2024-09-17T01:55:13.389+0000 7f5ee1ddd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:55:13.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[36645]: 2024-09-17T01:55:13.389+0000 7f5ee1ddd700 -1 osd.7 148 *** Got signal Terminated *** 2024-09-17T01:55:13.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:13 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[36645]: 2024-09-17T01:55:13.389+0000 7f5ee1ddd700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:55:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:13] "GET /metrics HTTP/1.1" 200 38315 "" "Prometheus/2.43.0" 2024-09-17T01:55:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:13 smithi105 ceph-mon[63427]: osd.7 marked itself down and dead 2024-09-17T01:55:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:13 smithi081 ceph-mon[77854]: osd.7 marked itself down and dead 2024-09-17T01:55:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:13 smithi081 ceph-mon[80183]: osd.7 marked itself down and dead 2024-09-17T01:55:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:13 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:14.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:14 smithi105 podman[87041]: 2024-09-17 01:55:14.451555071 +0000 UTC m=+1.161486595 container died 2fb3cc0892e6a47b111e02c03f5ea717766803356ae8261610d3d4bbfc6c983b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, io.openshift.tags=base centos centos-stream, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, version=8, distribution-scope=public, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git) 2024-09-17T01:55:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:14 smithi105 ceph-mon[63427]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:14 smithi105 ceph-mon[63427]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:55:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:14 smithi105 ceph-mon[63427]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T01:55:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:14 smithi105 ceph-mon[63427]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T01:55:15.137 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:14 smithi105 podman[87041]: 2024-09-17 01:55:14.847886934 +0000 UTC m=+1.557818531 container cleanup 2fb3cc0892e6a47b111e02c03f5ea717766803356ae8261610d3d4bbfc6c983b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc.) 2024-09-17T01:55:15.137 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:14 smithi105 bash[87041]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7 2024-09-17T01:55:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[77854]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[77854]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:55:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[77854]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T01:55:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[77854]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T01:55:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[80183]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[80183]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:55:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[80183]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T01:55:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:14 smithi081 ceph-mon[80183]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T01:55:15.635 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:15 smithi105 podman[87058]: 2024-09-17 01:55:15.233299493 +0000 UTC m=+0.776541450 container remove 2fb3cc0892e6a47b111e02c03f5ea717766803356ae8261610d3d4bbfc6c983b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, vcs-type=git, io.k8s.display-name=CentOS Stream 8, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, RELEASE=HEAD, io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-17T01:55:15.988 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:15 smithi105 podman[87145]: 2024-09-17 01:55:15.616107776 +0000 UTC m=+0.016195213 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:15.988 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:15 smithi105 podman[87145]: 2024-09-17 01:55:15.733925575 +0000 UTC m=+0.134013002 container create 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-17T01:55:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:15 smithi105 ceph-mon[63427]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T01:55:16.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 podman[87145]: 2024-09-17 01:55:16.011908648 +0000 UTC m=+0.411996075 container init 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T01:55:16.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 podman[87145]: 2024-09-17 01:55:16.015440188 +0000 UTC m=+0.415527622 container start 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-17T01:55:16.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 podman[87145]: 2024-09-17 01:55:16.094940229 +0000 UTC m=+0.495027659 container attach 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:55:16.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 podman[87145]: 2024-09-17 01:55:16.200951641 +0000 UTC m=+0.601039079 container died 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:55:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:15 smithi081 ceph-mon[77854]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T01:55:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:15 smithi081 ceph-mon[80183]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T01:55:16.990 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 podman[87168]: 2024-09-17 01:55:16.773721211 +0000 UTC m=+0.566655177 container remove 51cff918e5c81a0c8bc939be0cb2e5cec93555163411ea2f0ef5384361830b8f (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T01:55:16.990 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T01:55:16.990 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 systemd[1]: Stopped Ceph osd.7 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:55:16.990 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.7.service: Consumed 16.703s CPU time. 2024-09-17T01:55:17.308 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:16 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:17.309 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:16 smithi105 ceph-mon[63427]: pgmap v249: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:55:17.309 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:16 smithi105 systemd[1]: Starting Ceph osd.7 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:55:17.309 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 podman[87268]: 2024-09-17 01:55:17.19981209 +0000 UTC m=+0.107240485 container create 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T01:55:17.309 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 podman[87268]: 2024-09-17 01:55:17.108459084 +0000 UTC m=+0.015887491 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:16 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:17.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:16 smithi081 ceph-mon[77854]: pgmap v249: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:55:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:16 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:16 smithi081 ceph-mon[80183]: pgmap v249: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-09-17T01:55:17.566 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 podman[87268]: 2024-09-17 01:55:17.419535559 +0000 UTC m=+0.326963958 container init 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 podman[87268]: 2024-09-17 01:55:17.422821752 +0000 UTC m=+0.330250156 container start 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 podman[87268]: 2024-09-17 01:55:17.519606652 +0000 UTC m=+0.427035064 container attach 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 bash[87268]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:55:17.567 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:17 smithi105 bash[87268]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:55:18.313 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-mon[63427]: pgmap v250: 161 pgs: 37 active+undersized, 3 stale+active+clean, 25 active+undersized+degraded, 96 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:55:18.314 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-mon[63427]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:55:18.314 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:55:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:18 smithi081 ceph-mon[77854]: pgmap v250: 161 pgs: 37 active+undersized, 3 stale+active+clean, 25 active+undersized+degraded, 96 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:55:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:18 smithi081 ceph-mon[77854]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:18.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:18 smithi081 ceph-mon[80183]: pgmap v250: 161 pgs: 37 active+undersized, 3 stale+active+clean, 25 active+undersized+degraded, 96 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-09-17T01:55:18.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:18 smithi081 ceph-mon[80183]: Health check failed: Degraded data redundancy: 96/723 objects degraded (13.278%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate[87282]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 bash[87268]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T01:55:18.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 podman[87268]: 2024-09-17 01:55:18.344829312 +0000 UTC m=+1.252257717 container died 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-17T01:55:19.303 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:18 smithi105 podman[87407]: 2024-09-17 01:55:18.97379219 +0000 UTC m=+0.618996593 container remove 325707ef806e2df5424d2b2df2255293309f66183ceb70034aa85a292751c49e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:55:19.303 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 podman[87466]: 2024-09-17 01:55:19.084088025 +0000 UTC m=+0.015997134 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:19.303 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 podman[87466]: 2024-09-17 01:55:19.19141308 +0000 UTC m=+0.123322181 container create 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T01:55:19.885 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 podman[87466]: 2024-09-17 01:55:19.410613058 +0000 UTC m=+0.342522164 container init 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T01:55:19.885 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 podman[87466]: 2024-09-17 01:55:19.413987401 +0000 UTC m=+0.345896501 container start 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T01:55:19.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 ceph-osd[87485]: -- 172.21.15.105:0/902576726 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e0fa94fa40 con 0x55e0fb716400 2024-09-17T01:55:19.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 bash[87466]: 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 2024-09-17T01:55:19.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:19 smithi105 systemd[1]: Started Ceph osd.7 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:55:20.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:20 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:20 smithi105 ceph-mon[63427]: pgmap v251: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:20.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[77854]: pgmap v251: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:20.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:20.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:20 smithi081 ceph-mon[80183]: pgmap v251: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:21.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:21 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:55:21.046+0000 7fbf325a8640 -1 Falling back to public interface 2024-09-17T01:55:23.090 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:22 smithi105 ceph-mon[63427]: pgmap v252: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:22 smithi081 ceph-mon[77854]: pgmap v252: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:22 smithi081 ceph-mon[80183]: pgmap v252: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:23] "GET /metrics HTTP/1.1" 200 38320 "" "Prometheus/2.43.0" 2024-09-17T01:55:24.871 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:55:24.601+0000 7fbf325a8640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T01:55:24.871 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:55:24.636+0000 7fbf325a8640 -1 osd.7 148 log_to_monitors true 2024-09-17T01:55:24.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.872 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.872 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-mon[63427]: pgmap v253: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:24.872 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.872 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:24 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[77854]: pgmap v253: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[80183]: pgmap v253: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:24.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:24 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:25.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:25 smithi105 ceph-mon[63427]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:25.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:25 smithi105 ceph-mon[63427]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:25.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:25 smithi081 ceph-mon[77854]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:25.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:25 smithi081 ceph-mon[77854]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:25.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:25 smithi081 ceph-mon[80183]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:25.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:25 smithi081 ceph-mon[80183]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:55:26.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:55:26.037+0000 7fbf2634b640 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:55:26.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:26.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:55:26.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T01:55:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: pgmap v255: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:26.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:26 smithi105 ceph-mon[63427]: from='osd.7 ' entity='osd.7' 2024-09-17T01:55:26.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: pgmap v255: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[77854]: from='osd.7 ' entity='osd.7' 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:55:26.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T01:55:26.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: from='osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T01:55:26.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: pgmap v255: 161 pgs: 43 active+undersized, 26 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 97/723 objects degraded (13.416%) 2024-09-17T01:55:26.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 97/723 objects degraded (13.416%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:26.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:26 smithi081 ceph-mon[80183]: from='osd.7 ' entity='osd.7' 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429] boot 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:27.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:27 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429] boot 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: osd.7 [v2:172.21.15.105:6824/2885697429,v1:172.21.15.105:6825/2885697429] boot 2024-09-17T01:55:27.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:27.932 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:27 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: pgmap v257: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: pgmap v258: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T01:55:28.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.889 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:55:28.890 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.892 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: pgmap v257: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: pgmap v258: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T01:55:28.932 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: pgmap v257: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: pgmap v258: 161 pgs: 10 peering, 38 active+undersized, 21 active+undersized+degraded, 92 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 75/723 objects degraded (10.373%) 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.933 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T01:55:28.934 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T01:55:28.935 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.936 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:55:28.937 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.938 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:55:28.939 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:55:28.940 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.941 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:55:28.942 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.943 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:55:28.944 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:28.945 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nfs 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: pgmap v260: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: pgmap v261: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: pgmap v262: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: pgmap v263: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:30.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nfs 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Finalizing container_image settings 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: Upgrade: Complete! 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: pgmap v260: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: pgmap v261: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: pgmap v262: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: pgmap v263: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nfs 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Finalizing container_image settings 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: Upgrade: Complete! 2024-09-17T01:55:30.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: pgmap v260: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: pgmap v261: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: pgmap v262: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: pgmap v263: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:30.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:32.089 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:31 smithi081 ceph-mon[77854]: pgmap v264: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:32.089 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:31 smithi081 ceph-mon[77854]: Health check update: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:32.089 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:31 smithi081 ceph-mon[80183]: pgmap v264: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:32.090 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:31 smithi081 ceph-mon[80183]: Health check update: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:32.115 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:55:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:31 smithi105 ceph-mon[63427]: pgmap v264: 161 pgs: 10 peering, 35 active+undersized, 20 active+undersized+degraded, 96 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 74/723 objects degraded (10.235%) 2024-09-17T01:55:32.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:31 smithi105 ceph-mon[63427]: Health check update: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T01:55:32.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:32 smithi081 ceph-mon[77854]: from='client.54567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:32.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:32 smithi081 ceph-mon[80183]: from='client.54567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:33.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:32 smithi105 ceph-mon[63427]: from='client.54567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:33.647 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:33] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-09-17T01:55:33.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:33 smithi081 ceph-mon[77854]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:33.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:33 smithi081 ceph-mon[77854]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded) 2024-09-17T01:55:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:33 smithi081 ceph-mon[80183]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:33.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:33 smithi081 ceph-mon[80183]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded) 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (21m) 2m ago 29m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (8m) 10s ago 29m 80.5M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 2m ago 28m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (8m) 10s ago 33m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:55:34.105 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (17m) 2m ago 35m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (6m) 2m ago 35m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (7m) 10s ago 34m 48.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (6m) 2m ago 34m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (21m) 2m ago 30m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (21m) 10s ago 30m 15.4M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (4m) 2m ago 33m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (4m) 2m ago 33m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (3m) 2m ago 32m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (2m) 2m ago 32m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (118s) 10s ago 32m 47.6M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (83s) 10s ago 31m 44.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (48s) 10s ago 31m 44.0M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (15s) 10s ago 30m 12.1M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:55:34.106 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (8m) 10s ago 30m 67.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:55:34.107 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (28m) 2m ago 28m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:55:34.107 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (28m) 10s ago 28m 81.7M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:55:34.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:33 smithi105 ceph-mon[63427]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:34.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:33 smithi105 ceph-mon[63427]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/723 objects degraded (10.235%), 20 pgs degraded) 2024-09-17T01:55:34.747 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-09-17T01:55:36.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:35 smithi105 ceph-mon[63427]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:36.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:35 smithi105 ceph-mon[63427]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:55:36.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:35 smithi081 ceph-mon[77854]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:36.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:35 smithi081 ceph-mon[77854]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:55:36.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:35 smithi081 ceph-mon[80183]: from='client.54573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:36.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:35 smithi081 ceph-mon[80183]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-09-17T01:55:36.871 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:55:37.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:36 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:37.153 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:36 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:37.153 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:36 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:37.565 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-09-17T01:55:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:37 smithi105 ceph-mon[63427]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-17T01:55:38.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:37 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3024301208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:38.166 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:37 smithi081 ceph-mon[77854]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-17T01:55:38.166 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:37 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3024301208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:38.166 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:37 smithi081 ceph-mon[80183]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 631 B/s rd, 0 op/s 2024-09-17T01:55:38.166 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:37 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3024301208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:39.607 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:55:40.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[77854]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:55:40.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:40.217 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/267314980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:40.217 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[80183]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:55:40.217 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:40.218 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:39 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/267314980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:40.219 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:55:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:39 smithi105 ceph-mon[63427]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:55:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:39 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:39 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/267314980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:42.135 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:41 smithi081 ceph-mon[77854]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:55:42.136 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:55:42.137 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:41 smithi081 ceph-mon[80183]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:41 smithi105 ceph-mon[63427]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:42.765 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:55:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:43] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-09-17T01:55:44.285 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[77854]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[77854]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[80183]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[80183]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:44.286 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:43 smithi105 ceph-mon[63427]: from='client.54591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:43 smithi105 ceph-mon[63427]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:44.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:44.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:44.773 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:55:44.774 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:55:44.774 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:55:45.076 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:44 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/145024652' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:55:45.077 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:44 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/145024652' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:55:45.362 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-09-17T01:55:45.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:44 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/145024652' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:55:46.287 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:45 smithi081 ceph-mon[77854]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:46.287 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:45 smithi081 ceph-mon[80183]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:45 smithi105 ceph-mon[63427]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:46 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:47.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:46 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:46 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:48.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:47 smithi081 ceph-mon[77854]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:48.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:47 smithi081 ceph-mon[80183]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:47 smithi105 ceph-mon[63427]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:55:48.710 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:49.251 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[77854]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[80183]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:49.252 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:48 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:49.276 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T01:55:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:48 smithi105 ceph-mon[63427]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:55:49.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:48 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[77854]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[77854]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[77854]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[77854]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[77854]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:50.309 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[80183]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.310 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[80183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:50.310 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[80183]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-17T01:55:50.310 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[80183]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.310 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:49 smithi081 ceph-mon[80183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:49 smithi105 ceph-mon[63427]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:49 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:50.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:49 smithi105 ceph-mon[63427]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-09-17T01:55:50.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:49 smithi105 ceph-mon[63427]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:50.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:49 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:55:51.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:55:51.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:55:51.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:51.546 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mgr 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.547 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.548 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[80183]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mgr 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:55:51.549 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.550 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:51 smithi081 ceph-mon[77854]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all osd 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mds 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:51.637 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:51 smithi105 ceph-mon[63427]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-09-17T01:55:53.043 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:55:53.559 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:55:53] "GET /metrics HTTP/1.1" 200 38326 "" "Prometheus/2.43.0" 2024-09-17T01:55:53.711 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (21m) 2m ago 29m 25.1M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (8m) 30s ago 29m 80.5M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 2m ago 28m 60.7M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (8m) 30s ago 34m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (18m) 2m ago 35m 503M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (6m) 2m ago 36m 53.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (7m) 30s ago 34m 48.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (6m) 2m ago 34m 41.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (22m) 2m ago 30m 15.0M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (21m) 30s ago 30m 15.4M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (5m) 2m ago 33m 45.9M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:55:53.712 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (4m) 2m ago 33m 58.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (3m) 2m ago 33m 49.3M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (2m) 2m ago 32m 11.8M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (2m) 30s ago 32m 47.6M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (103s) 30s ago 32m 44.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (68s) 30s ago 31m 44.0M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (34s) 30s ago 31m 12.1M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (8m) 30s ago 30m 67.7M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (29m) 2m ago 29m 79.8M - 17.2.0 e1d6a67b021e 57ce0c3ba234 2024-09-17T01:55:53.713 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (29m) 30s ago 29m 81.7M - 17.2.0 e1d6a67b021e 7921697dda0b 2024-09-17T01:55:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: Upgrade: Updating rgw.foo.smithi081.fnhdrk (1/2) 2024-09-17T01:55:53.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:55:53.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:53 smithi105 ceph-mon[63427]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:53.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: Upgrade: Updating rgw.foo.smithi081.fnhdrk (1/2) 2024-09-17T01:55:53.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:53.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:53.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[80183]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: Upgrade: Updating rgw.foo.smithi081.fnhdrk (1/2) 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi081.fnhdrk", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: Deploying daemon rgw.foo.smithi081.fnhdrk on smithi081 2024-09-17T01:55:53.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:53 smithi081 ceph-mon[77854]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:54.122 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:55:54.122 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:55:54.123 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T01:55:54.124 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 13 2024-09-17T01:55:54.124 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:55:54.124 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:55:54.513 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "0/2 daemons upgraded", 2024-09-17T01:55:54.514 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading rgw daemons", 2024-09-17T01:55:54.514 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:55:54.514 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:55:54.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:54 smithi105 ceph-mon[63427]: from='client.64576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:54 smithi105 ceph-mon[63427]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:54 smithi105 ceph-mon[63427]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:54 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2265107423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[77854]: from='client.64576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[77854]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[77854]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2265107423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:54.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[80183]: from='client.64576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[80183]: from='client.64582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[80183]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:54.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:54 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2265107423' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:55:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:55 smithi081 ceph-mon[77854]: from='client.64600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:55.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:55 smithi081 ceph-mon[77854]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:55 smithi081 ceph-mon[80183]: from='client.64600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:55.681 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:55 smithi081 ceph-mon[80183]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:55.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:55 smithi105 ceph-mon[63427]: from='client.64600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:55:55.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:55 smithi105 ceph-mon[63427]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:56.854 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:56 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:56.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:56 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:55:56.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:56 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:56.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:56 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:57.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:57 smithi105 ceph-mon[63427]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-17T01:55:57.930 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:57 smithi081 ceph-mon[77854]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-17T01:55:57.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:57 smithi081 ceph-mon[80183]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: Upgrade: Updating rgw.foo.smithi105.ttsowb (2/2) 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:55:59.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:59.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: Upgrade: Updating rgw.foo.smithi105.ttsowb (2/2) 2024-09-17T01:55:59.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:59.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: Upgrade: Updating rgw.foo.smithi105.ttsowb (2/2) 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.ttsowb", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: Deploying daemon rgw.foo.smithi105.ttsowb on smithi105 2024-09-17T01:55:59.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:00.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:55:59 smithi105 ceph-mon[63427]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-17T01:56:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:55:59 smithi081 ceph-mon[77854]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-17T01:56:00.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:55:59 smithi081 ceph-mon[80183]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 0 B/s wr, 94 op/s 2024-09-17T01:56:02.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:01 smithi105 ceph-mon[63427]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 98 op/s 2024-09-17T01:56:02.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:01 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:02.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:01 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:02.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[77854]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 98 op/s 2024-09-17T01:56:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:02.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:02.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[80183]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 98 op/s 2024-09-17T01:56:02.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:02.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:01 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:03.759 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:03] "GET /metrics HTTP/1.1" 200 38325 "" "Prometheus/2.43.0" 2024-09-17T01:56:04.108 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:03 smithi105 ceph-mon[63427]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 85 B/s wr, 199 op/s 2024-09-17T01:56:04.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:03 smithi081 ceph-mon[77854]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 85 B/s wr, 199 op/s 2024-09-17T01:56:04.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:03 smithi081 ceph-mon[80183]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 85 B/s wr, 199 op/s 2024-09-17T01:56:06.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 85 B/s wr, 198 op/s 2024-09-17T01:56:06.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:05 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 85 B/s wr, 198 op/s 2024-09-17T01:56:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 85 B/s wr, 198 op/s 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:06.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:05 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.053 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:06 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:07.053 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.053 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:06 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.150 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:07.151 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.151 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.151 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:07.151 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:07.151 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:06 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:08.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:07 smithi105 ceph-mon[63427]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:08.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:07 smithi081 ceph-mon[77854]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:08.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:07 smithi081 ceph-mon[80183]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]: dispatch 2024-09-17T01:56:10.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]': finished 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]': finished 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:56:10.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.140 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:09 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]': finished 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]': finished 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 246 op/s 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.183 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi081.fnhdrk"}]': finished 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]: dispatch 2024-09-17T01:56:10.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.ttsowb"}]': finished 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.185 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:56:10.186 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.187 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:56:10.188 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:56:10.189 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.190 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:56:10.191 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:56:10.192 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:56:10.192 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:10.192 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:10.192 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:09 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 195 B/s wr, 194 op/s 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 242 B/s wr, 212 op/s 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nfs 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Finalizing container_image settings 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Upgrade: Complete! 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 152 B/s wr, 88 op/s 2024-09-17T01:56:11.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 217 B/s wr, 126 op/s 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: Checking dashboard <-> RGW credentials 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:56:11.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:10 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 195 B/s wr, 194 op/s 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 242 B/s wr, 212 op/s 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nfs 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Finalizing container_image settings 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Upgrade: Complete! 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 152 B/s wr, 88 op/s 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 217 B/s wr, 126 op/s 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: Checking dashboard <-> RGW credentials 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 195 B/s wr, 194 op/s 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 135 KiB/s rd, 242 B/s wr, 212 op/s 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nfs 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Finalizing container_image settings 2024-09-17T01:56:11.182 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Upgrade: Complete! 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 152 B/s wr, 88 op/s 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 217 B/s wr, 126 op/s 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: Checking dashboard <-> RGW credentials 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.6 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:56:11.183 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:10 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:13.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:12 smithi105 ceph-mon[63427]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 5 op/s 2024-09-17T01:56:13.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:12 smithi081 ceph-mon[77854]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 5 op/s 2024-09-17T01:56:13.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:12 smithi081 ceph-mon[80183]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 5 op/s 2024-09-17T01:56:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:13] "GET /metrics HTTP/1.1" 200 38325 "" "Prometheus/2.43.0" 2024-09-17T01:56:15.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[77854]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-17T01:56:15.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:15.099 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:15.100 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[80183]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-17T01:56:15.100 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:15.100 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:14 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:15.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:14 smithi105 ceph-mon[63427]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 6 op/s 2024-09-17T01:56:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:15.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:14 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:17.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:16 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:17.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:16 smithi105 ceph-mon[63427]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-09-17T01:56:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:16 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:16 smithi081 ceph-mon[80183]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-09-17T01:56:17.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:16 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:17.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:16 smithi081 ceph-mon[77854]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-09-17T01:56:19.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:18 smithi105 ceph-mon[63427]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-17T01:56:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:18 smithi081 ceph-mon[77854]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-17T01:56:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:18 smithi081 ceph-mon[80183]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 4 op/s 2024-09-17T01:56:21.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:20 smithi105 ceph-mon[63427]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-17T01:56:21.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:20 smithi081 ceph-mon[80183]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-17T01:56:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:20 smithi081 ceph-mon[77854]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-09-17T01:56:23.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:22 smithi105 ceph-mon[63427]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-17T01:56:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:22 smithi081 ceph-mon[80183]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-17T01:56:23.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:22 smithi081 ceph-mon[77854]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-09-17T01:56:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:23] "GET /metrics HTTP/1.1" 200 38358 "" "Prometheus/2.43.0" 2024-09-17T01:56:25.110 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:24 smithi081 ceph-mon[77854]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:56:25.110 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:24 smithi081 ceph-mon[80183]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:56:25.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:24 smithi105 ceph-mon[63427]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T01:56:25.594 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:56:26.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:25 smithi105 ceph-mon[63427]: from='client.54708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:26.141 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:25 smithi081 ceph-mon[77854]: from='client.54708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:26.142 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:25 smithi081 ceph-mon[80183]: from='client.54708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:26.989 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:26 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:26.989 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:26 smithi081 ceph-mon[77854]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:26.989 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:26 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:26.990 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:26 smithi081 ceph-mon[80183]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:27.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:26 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:27.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:26 smithi105 ceph-mon[63427]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:27.707 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:56:27.707 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (22m) 22s ago 30m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (9m) 22s ago 30m 81.0M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 22s ago 29m 61.0M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (9m) 22s ago 34m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (18m) 22s ago 36m 508M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (7m) 22s ago 36m 61.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (8m) 22s ago 35m 53.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (7m) 22s ago 35m 46.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (22m) 22s ago 31m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (22m) 22s ago 31m 15.8M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (5m) 22s ago 34m 54.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (5m) 22s ago 34m 67.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:56:27.708 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (4m) 22s ago 33m 55.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (3m) 22s ago 33m 52.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (2m) 22s ago 32m 52.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (2m) 22s ago 32m 48.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (102s) 22s ago 32m 47.3M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (68s) 22s ago 31m 55.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (9m) 22s ago 31m 68.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (31s) 22s ago 29m 76.0M - 19.1.1-330-gc2ddf972 ebb0615149de 0f90aadb4099 2024-09-17T01:56:27.709 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (26s) 22s ago 29m 77.0M - 19.1.1-330-gc2ddf972 ebb0615149de bc82a4219daa 2024-09-17T01:56:28.452 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[77854]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[77854]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[80183]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[80183]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:29.047 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:29.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:28 smithi105 ceph-mon[63427]: from='client.54714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:28 smithi105 ceph-mon[63427]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:29.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:30.536 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:56:31.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:30 smithi105 ceph-mon[63427]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:31.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:30 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3525871110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:31.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:30 smithi081 ceph-mon[77854]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:31.142 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:30 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3525871110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:31.142 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:30 smithi081 ceph-mon[80183]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:31.142 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:30 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3525871110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:31.166 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-09-17T01:56:33.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:32 smithi105 ceph-mon[63427]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:33.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:32 smithi081 ceph-mon[77854]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:33.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:32 smithi081 ceph-mon[80183]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:33.189 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:56:33.840 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:33] "GET /metrics HTTP/1.1" 200 38360 "" "Prometheus/2.43.0" 2024-09-17T01:56:33.872 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:56:34.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:33 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1790733716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:34.168 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:33 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1790733716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:34.168 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:33 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1790733716' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:35.120 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:34 smithi081 ceph-mon[77854]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:35.121 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:34 smithi081 ceph-mon[80183]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:35.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:34 smithi105 ceph-mon[63427]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:35.956 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:56:35.956 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": null, 2024-09-17T01:56:35.956 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": false, 2024-09-17T01:56:35.956 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "", 2024-09-17T01:56:35.956 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [], 2024-09-17T01:56:35.957 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": null, 2024-09-17T01:56:35.957 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "", 2024-09-17T01:56:35.957 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:56:35.957 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:56:36.730 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:56:37.032 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:37.032 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[77854]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:37.032 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[77854]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:37.033 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:37.033 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[80183]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:37.033 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:36 smithi081 ceph-mon[80183]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:37.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:36 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:37.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:36 smithi105 ceph-mon[63427]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:37.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:36 smithi105 ceph-mon[63427]: from='client.54732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:38.851 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:56:38.934 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:56:38.934 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:56:39.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:38 smithi105 ceph-mon[63427]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:39.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:38 smithi081 ceph-mon[77854]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:39.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:38 smithi081 ceph-mon[80183]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:39.535 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T01:56:40.073 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:39 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3150020406' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:40.074 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:39 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3150020406' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:40.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:39 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3150020406' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:41.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:40 smithi105 ceph-mon[63427]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:41.170 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:40 smithi081 ceph-mon[77854]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:41.170 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:40 smithi081 ceph-mon[80183]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:41.776 INFO:teuthology.orchestra.run.smithi081.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:42.417 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:56:42.427 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T01:56:42.427 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: from='client.64720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:42 smithi105 ceph-mon[63427]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: from='client.64720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[77854]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: from='client.64720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-09-17T01:56:43.184 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:43.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:43.185 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:42 smithi081 ceph-mon[80183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:56:43.777 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:43] "GET /metrics HTTP/1.1" 200 38360 "" "Prometheus/2.43.0" 2024-09-17T01:56:44.879 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.879 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:56:44.879 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:56:44.879 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:44.879 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mgr 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mon 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all crash 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all osd 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.880 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all mds 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.881 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mgr 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mon 2024-09-17T01:56:44.882 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all crash 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all osd 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all mds 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.883 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:44.884 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (squid) 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mgr 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mon 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all crash 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all osd 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all mds 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rgw 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.887 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:44.888 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:46.529 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:56:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:46.930 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: Upgrade: Updating iscsi.foo.smithi081.umxnyg 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[80183]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: Upgrade: Updating iscsi.foo.smithi081.umxnyg 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:46.931 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:46 smithi081 ceph-mon[77854]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:56:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-09-17T01:56:47.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: Upgrade: Updating iscsi.foo.smithi081.umxnyg 2024-09-17T01:56:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:56:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi081.umxnyg", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T01:56:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:56:47.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:46 smithi105 ceph-mon[63427]: Deploying daemon iscsi.foo.smithi081.umxnyg on smithi081 2024-09-17T01:56:47.206 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:56:47.206 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (22m) 41s ago 30m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:56:47.206 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (9m) 42s ago 30m 81.0M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:56:47.206 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 41s ago 29m 61.0M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:56:47.206 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (9m) 42s ago 35m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (19m) 41s ago 36m 508M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (7m) 41s ago 37m 61.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (8m) 42s ago 35m 53.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (7m) 41s ago 35m 46.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (22m) 41s ago 31m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (22m) 42s ago 31m 15.8M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (6m) 41s ago 34m 54.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (5m) 41s ago 34m 67.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (4m) 41s ago 34m 55.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (3m) 41s ago 33m 52.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (3m) 42s ago 33m 52.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (2m) 42s ago 32m 48.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (2m) 42s ago 32m 47.3M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:56:47.207 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (88s) 42s ago 32m 55.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:56:47.208 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (9m) 42s ago 31m 68.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:56:47.208 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (51s) 41s ago 29m 76.0M - 19.1.1-330-gc2ddf972 ebb0615149de 0f90aadb4099 2024-09-17T01:56:47.208 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (46s) 42s ago 29m 77.0M - 19.1.1-330-gc2ddf972 ebb0615149de bc82a4219daa 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:56:47.606 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 15 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:56:47.607 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [ 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "osd", 2024-09-17T01:56:47.999 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr", 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw", 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: "mon" 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "15/21 daemons upgraded", 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Currently upgrading iscsi daemons", 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": false 2024-09-17T01:56:48.000 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:56:48.102 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[77854]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.102 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[77854]: from='client.54756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.103 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2222311193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:48.103 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[80183]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.103 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[80183]: from='client.54756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.103 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:47 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2222311193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:48.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:47 smithi105 ceph-mon[63427]: from='client.54750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:47 smithi105 ceph-mon[63427]: from='client.54756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:48.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:47 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2222311193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:56:48.424 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T01:56:48.424 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:56:48.424 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:56:49.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:48 smithi105 ceph-mon[63427]: from='client.54762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:48 smithi105 ceph-mon[63427]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:49.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:48 smithi105 ceph-mon[63427]: from='client.64750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:48 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2712913689' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:49.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[77854]: from='client.54762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[77854]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[77854]: from='client.64750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2712913689' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[80183]: from='client.54762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[80183]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:49.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[80183]: from='client.64750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:56:49.182 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:48 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2712913689' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:56:51.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:50 smithi105 ceph-mon[63427]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:51.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:50 smithi081 ceph-mon[77854]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:51.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:50 smithi081 ceph-mon[80183]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:53.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:52 smithi105 ceph-mon[63427]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:56:53.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:52 smithi081 ceph-mon[77854]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:56:53.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:52 smithi081 ceph-mon[80183]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:56:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:56:53] "GET /metrics HTTP/1.1" 200 38359 "" "Prometheus/2.43.0" 2024-09-17T01:56:55.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:54 smithi105 ceph-mon[63427]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:55.141 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:54 smithi081 ceph-mon[77854]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:55.141 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:54 smithi081 ceph-mon[80183]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:56 smithi105 ceph-mon[63427]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:57.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:56 smithi105 ceph-mon[63427]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:57.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:56 smithi081 ceph-mon[77854]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:57.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:56 smithi081 ceph-mon[77854]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:56 smithi081 ceph-mon[80183]: from='client.25213 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:56:57.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:56 smithi081 ceph-mon[80183]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:56:59.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:58 smithi105 ceph-mon[63427]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:59.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:59.169 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:58 smithi081 ceph-mon[77854]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:59.169 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:59.169 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:58 smithi081 ceph-mon[80183]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:56:59.169 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:56:59.169 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:56:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:56:58.933+0000 7f468e4af640 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:57:00.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:00.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:56:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:00.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:00.432 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:56:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:01.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:00 smithi105 ceph-mon[63427]: Health check failed: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-17T01:57:01.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:00 smithi081 ceph-mon[80183]: Health check failed: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-17T01:57:01.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:00 smithi081 ceph-mon[77854]: Health check failed: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-09-17T01:57:02.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:01 smithi105 ceph-mon[63427]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-09-17T01:57:02.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:01 smithi081 ceph-mon[80183]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-09-17T01:57:02.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:01 smithi081 ceph-mon[77854]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-09-17T01:57:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:03] "GET /metrics HTTP/1.1" 200 38356 "" "Prometheus/2.43.0" 2024-09-17T01:57:04.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:03 smithi105 ceph-mon[63427]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-17T01:57:04.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:03 smithi081 ceph-mon[77854]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-17T01:57:04.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:03 smithi081 ceph-mon[80183]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 858 B/s rd, 0 op/s 2024-09-17T01:57:06.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:05 smithi105 ceph-mon[63427]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-09-17T01:57:06.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:05 smithi081 ceph-mon[77854]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-09-17T01:57:06.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:05 smithi081 ceph-mon[80183]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 143 B/s rd, 0 op/s 2024-09-17T01:57:08.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:07 smithi105 ceph-mon[63427]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T01:57:08.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:07 smithi081 ceph-mon[77854]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T01:57:08.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:07 smithi081 ceph-mon[80183]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T01:57:10.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:09 smithi105 ceph-mon[63427]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T01:57:10.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:09 smithi081 ceph-mon[77854]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T01:57:10.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:09 smithi081 ceph-mon[80183]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T01:57:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:10 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/126093608' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:57:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:10 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/751711588' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:11.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:10 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/126093608' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:57:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/751711588' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:11.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/126093608' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T01:57:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/751711588' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:11.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:10 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]: dispatch 2024-09-17T01:57:12.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:11 smithi105 ceph-mon[63427]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:57:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:11 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]': finished 2024-09-17T01:57:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:11 smithi105 ceph-mon[63427]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T01:57:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:11 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3606577599' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:12.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:11 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:12.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[77854]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]': finished 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[77854]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3606577599' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[80183]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/3745730292"}]': finished 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[80183]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3606577599' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:12.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:11 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]: dispatch 2024-09-17T01:57:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:12 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]': finished 2024-09-17T01:57:13.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:12 smithi105 ceph-mon[63427]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T01:57:13.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:12 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1209229301' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:12 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]': finished 2024-09-17T01:57:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[77854]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T01:57:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1209229301' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6800/3438292770"}]': finished 2024-09-17T01:57:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[80183]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T01:57:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1209229301' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:12 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]: dispatch 2024-09-17T01:57:13.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:13] "GET /metrics HTTP/1.1" 200 38356 "" "Prometheus/2.43.0" 2024-09-17T01:57:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-17T01:57:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]': finished 2024-09-17T01:57:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T01:57:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]: dispatch 2024-09-17T01:57:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:14.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-17T01:57:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]': finished 2024-09-17T01:57:14.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]: dispatch 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1578344991"}]': finished 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]: dispatch 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:57:14.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:13 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:14 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]': finished 2024-09-17T01:57:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:14 smithi105 ceph-mon[63427]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T01:57:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:14 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/3246907786' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:15.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:14 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]': finished 2024-09-17T01:57:15.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[77854]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/3246907786' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1942591421' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:6801/3438292770"}]': finished 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[80183]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/3246907786' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:15.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:14 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]: dispatch 2024-09-17T01:57:16.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:15 smithi105 ceph-mon[63427]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:57:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:15 smithi105 ceph-mon[63427]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]': finished 2024-09-17T01:57:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:15 smithi105 ceph-mon[63427]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T01:57:16.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:15 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]: dispatch 2024-09-17T01:57:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[77854]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:57:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[77854]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]': finished 2024-09-17T01:57:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[77854]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T01:57:16.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]: dispatch 2024-09-17T01:57:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[80183]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T01:57:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[80183]: from='client.? ' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/1565080892"}]': finished 2024-09-17T01:57:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[80183]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T01:57:16.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:15 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]: dispatch 2024-09-17T01:57:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:17 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]': finished 2024-09-17T01:57:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:17 smithi105 ceph-mon[63427]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T01:57:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:17 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]': finished 2024-09-17T01:57:17.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:17 smithi081 ceph-mon[77854]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T01:57:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:17 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4227137903' entity='client.iscsi.foo.smithi081.umxnyg' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.81:0/2286991754"}]': finished 2024-09-17T01:57:17.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:17 smithi081 ceph-mon[80183]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T01:57:18.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:18 smithi105 ceph-mon[63427]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:18.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:18 smithi081 ceph-mon[77854]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:18.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:18 smithi081 ceph-mon[80183]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:18.785 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:57:19.121 INFO:teuthology.orchestra.run.smithi081.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed." 2024-09-17T01:57:19.871 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:57:20.294 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:20 smithi081 ceph-mon[77854]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:20.294 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:20 smithi081 ceph-mon[77854]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-09-17T01:57:20.295 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:20 smithi081 ceph-mon[80183]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:20.295 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:20 smithi081 ceph-mon[80183]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-09-17T01:57:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:20 smithi105 ceph-mon[63427]: from='client.64822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:20.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:20 smithi105 ceph-mon[63427]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-09-17T01:57:21.279 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:21 smithi081 ceph-mon[77854]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:21.279 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:21 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:21.279 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:21 smithi081 ceph-mon[80183]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:21.279 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:21 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:21 smithi105 ceph-mon[63427]: from='client.64828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:21.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:21 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (23m) 76s ago 31m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (9m) 76s ago 31m 81.0M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 76s ago 30m 61.0M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (10m) 76s ago 35m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (19m) 76s ago 37m 508M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (8m) 76s ago 37m 61.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (9m) 76s ago 35m 53.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (7m) 76s ago 36m 46.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (23m) 76s ago 32m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:57:21.891 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (22m) 76s ago 31m 15.8M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (6m) 76s ago 35m 54.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (6m) 76s ago 35m 67.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (5m) 76s ago 34m 55.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (4m) 76s ago 34m 52.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (3m) 76s ago 33m 52.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (3m) 76s ago 33m 48.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (2m) 76s ago 33m 47.3M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (2m) 76s ago 32m 55.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (10m) 76s ago 32m 68.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (85s) 76s ago 30m 76.0M - 19.1.1-330-gc2ddf972 ebb0615149de 0f90aadb4099 2024-09-17T01:57:21.892 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (80s) 76s ago 30m 77.0M - 19.1.1-330-gc2ddf972 ebb0615149de bc82a4219daa 2024-09-17T01:57:22.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:22 smithi081 ceph-mon[77854]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-09-17T01:57:22.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:22 smithi081 ceph-mon[80183]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-09-17T01:57:22.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:22 smithi105 ceph-mon[63427]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-09-17T01:57:22.550 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:57:23.162 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:23 smithi081 ceph-mon[77854]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:23.163 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:23 smithi081 ceph-mon[80183]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:23 smithi105 ceph-mon[63427]: from='client.54858 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:57:23.774 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:23] "GET /metrics HTTP/1.1" 200 38435 "" "Prometheus/2.43.0" 2024-09-17T01:57:24.029 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:24 smithi081 ceph-mon[77854]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:24.029 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:24 smithi081 ceph-mon[80183]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:24.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:24 smithi105 ceph-mon[63427]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:57:24.529 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 15 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:57:24.530 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:57:24.886 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 01:57:24 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=cleanup t=2024-09-17T01:57:24.548483807Z level=info msg="Completed cleanup jobs" duration=2.328805ms 2024-09-17T01:57:25.148 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:25 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/727158371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:57:25.148 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:25 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/727158371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:57:25.170 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T01:57:25.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:25 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/727158371' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:57:26.365 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:26 smithi081 ceph-mon[77854]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:57:26.366 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:26 smithi081 ceph-mon[80183]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:57:26.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:26 smithi105 ceph-mon[63427]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T01:57:26.968 INFO:teuthology.orchestra.run.smithi081.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T01:57:27.587 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 60' 2024-09-17T01:57:28.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:28 smithi081 ceph-mon[77854]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:28.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:28 smithi081 ceph-mon[80183]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:28.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:28 smithi105 ceph-mon[63427]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T01:57:29.268 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:29 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:29.268 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:29 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:29.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:29 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:30.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:30 smithi105 ceph-mon[63427]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:30.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:30 smithi081 ceph-mon[77854]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:30.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:30 smithi081 ceph-mon[80183]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:31.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:31 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:31.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:31 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:31.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:31 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:32.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:32 smithi105 ceph-mon[63427]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:32.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:32 smithi081 ceph-mon[77854]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:32.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:32 smithi081 ceph-mon[80183]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:33 smithi105 ceph-mon[63427]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:33.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:33 smithi081 ceph-mon[77854]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:33.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:33 smithi081 ceph-mon[80183]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:33.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:33] "GET /metrics HTTP/1.1" 200 38438 "" "Prometheus/2.43.0" 2024-09-17T01:57:36.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:36 smithi081 ceph-mon[77854]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:36.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:36 smithi081 ceph-mon[80183]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:36.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:36 smithi105 ceph-mon[63427]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:38.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:38 smithi105 ceph-mon[63427]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:38.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:38 smithi081 ceph-mon[77854]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:38.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:38 smithi081 ceph-mon[80183]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:40.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:40 smithi105 ceph-mon[63427]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:40.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:40 smithi081 ceph-mon[77854]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:40.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:40 smithi081 ceph-mon[80183]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:41.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:41 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:41.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:41 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:41.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:41 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:42.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:42 smithi105 ceph-mon[63427]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:42.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:42 smithi081 ceph-mon[77854]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:42.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:42 smithi081 ceph-mon[80183]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:57:43.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:43] "GET /metrics HTTP/1.1" 200 38438 "" "Prometheus/2.43.0" 2024-09-17T01:57:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:44 smithi105 ceph-mon[63427]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:44.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:44 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:44 smithi081 ceph-mon[77854]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:44.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:44 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:44.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:44 smithi081 ceph-mon[80183]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:44.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:44 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:46.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:46 smithi105 ceph-mon[63427]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:46.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:46 smithi081 ceph-mon[77854]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:46.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:46 smithi081 ceph-mon[80183]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:48.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:48 smithi105 ceph-mon[63427]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:48.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:48 smithi081 ceph-mon[77854]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:48.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:48 smithi081 ceph-mon[80183]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:50 smithi105 ceph-mon[63427]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:50.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:50 smithi081 ceph-mon[77854]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:50.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:50 smithi081 ceph-mon[80183]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:51 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:51.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:51 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:51.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:51 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:57:52.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:52 smithi105 ceph-mon[63427]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:52.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:52 smithi081 ceph-mon[77854]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:52.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:52 smithi081 ceph-mon[80183]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:53.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:57:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:57:53] "GET /metrics HTTP/1.1" 200 38436 "" "Prometheus/2.43.0" 2024-09-17T01:57:54.381 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:54 smithi081 ceph-mon[77854]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:54.382 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:54 smithi081 ceph-mon[80183]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:54.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:54 smithi105 ceph-mon[63427]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:56.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:56 smithi105 ceph-mon[63427]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:56.431 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:56 smithi081 ceph-mon[77854]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:56.432 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:56 smithi081 ceph-mon[80183]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:57:57.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:57 smithi105 ceph-mon[63427]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:57.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:57 smithi081 ceph-mon[77854]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:57.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:57 smithi081 ceph-mon[80183]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:57:59.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:58 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:59.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:58 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:57:59.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:58 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:00.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:59 smithi105 ceph-mon[63427]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:00.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:58:00.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:58:00.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:57:59 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:58:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[77854]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:58:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:58:00.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:58:00.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[80183]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:00.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:58:00.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:58:00.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:57:59 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' 2024-09-17T01:58:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:00 smithi105 ceph-mon[63427]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-17T01:58:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:00 smithi105 ceph-mon[63427]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:00 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:01.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[77854]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-17T01:58:01.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[77854]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:01.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:01.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[80183]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-09-17T01:58:01.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[80183]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:01.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:00 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:03.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:02 smithi105 ceph-mon[63427]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:03.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:02 smithi081 ceph-mon[77854]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:03.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:02 smithi081 ceph-mon[80183]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:03.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:03 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:03] "GET /metrics HTTP/1.1" 200 38437 "" "Prometheus/2.43.0" 2024-09-17T01:58:05.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:04 smithi105 ceph-mon[63427]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:05.175 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:04 smithi081 ceph-mon[77854]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:05.175 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:04 smithi081 ceph-mon[80183]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:07.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:06 smithi081 ceph-mon[77854]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:07.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:06 smithi081 ceph-mon[80183]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:07.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:06 smithi105 ceph-mon[63427]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-09-17T01:58:09.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:08 smithi081 ceph-mon[77854]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:09.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:08 smithi081 ceph-mon[80183]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:09.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:08 smithi105 ceph-mon[63427]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:11.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:10 smithi081 ceph-mon[80183]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:58:11.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:10 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:10 smithi081 ceph-mon[77854]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:58:11.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:10 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:10 smithi105 ceph-mon[63427]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T01:58:11.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:10 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:13.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:12 smithi081 ceph-mon[80183]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:13.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:12 smithi081 ceph-mon[77854]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:13.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:12 smithi105 ceph-mon[63427]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:13.901 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:13 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:13] "GET /metrics HTTP/1.1" 200 38437 "" "Prometheus/2.43.0" 2024-09-17T01:58:14.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:13 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:14.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:13 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:14.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:13 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:15.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:14 smithi081 ceph-mon[80183]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:15.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:14 smithi081 ceph-mon[77854]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:15.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:14 smithi105 ceph-mon[63427]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:17.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:16 smithi081 ceph-mon[77854]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:17.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:16 smithi081 ceph-mon[80183]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:17.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:16 smithi105 ceph-mon[63427]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:19.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:18 smithi081 ceph-mon[77854]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:19.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:18 smithi081 ceph-mon[80183]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:19.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:18 smithi105 ceph-mon[63427]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:21.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:20 smithi081 ceph-mon[80183]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:21.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:20 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:20 smithi081 ceph-mon[77854]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:21.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:20 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:21.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:20 smithi105 ceph-mon[63427]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:21.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:20 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:23.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:22 smithi081 ceph-mon[77854]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:23.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:22 smithi081 ceph-mon[80183]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:23.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:22 smithi105 ceph-mon[63427]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:23.930 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:23 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:23] "GET /metrics HTTP/1.1" 200 38437 "" "Prometheus/2.43.0" 2024-09-17T01:58:25.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:24 smithi081 ceph-mon[77854]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:25.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:24 smithi081 ceph-mon[80183]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:25.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:24 smithi105 ceph-mon[63427]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:27.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:26 smithi081 ceph-mon[77854]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:27.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:26 smithi081 ceph-mon[80183]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:27.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:26 smithi105 ceph-mon[63427]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:29.180 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:28 smithi081 ceph-mon[80183]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:29.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:28 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:28 smithi081 ceph-mon[77854]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:29.181 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:28 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:29.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:28 smithi105 ceph-mon[63427]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:29.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:28 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:30.102 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:58:31.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:30 smithi081 ceph-mon[77854]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:31.097 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:30 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:31.097 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:30 smithi081 ceph-mon[80183]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:31.097 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:30 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:31.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:30 smithi105 ceph-mon[63427]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T01:58:31.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:30 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:32.118 INFO:teuthology.orchestra.run.smithi081.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:alertmanager.a smithi081 *:9093,9094 running (24m) 2m ago 32m 25.3M - 0.25.0 c8568f914cd2 dc15deba3321 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:grafana.a smithi105 *:3000 running (11m) 2m ago 32m 81.0M - 9.4.12 2bacad6d85d8 16b0af893ffa 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:iscsi.foo.smithi081.umxnyg smithi081 error 2m ago 31m 61.0M - 3.5 e1d6a67b021e 168b183d3250 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.x smithi105 *:8443,9283,8765 running (11m) 2m ago 36m 442M - 19.1.1-330-gc2ddf972 ebb0615149de a4d2bd4a6303 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:mgr.y smithi081 *:8443,9283,8765 running (20m) 2m ago 38m 508M - 19.1.1-330-gc2ddf972 ebb0615149de a7ee288e6ac2 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:mon.a smithi081 running (9m) 2m ago 38m 61.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 1dd8805b728c 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:mon.b smithi105 running (10m) 2m ago 37m 53.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 02f027eec663 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:mon.c smithi081 running (9m) 2m ago 37m 46.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de c257f6ba0121 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.a smithi081 *:9100 running (24m) 2m ago 33m 15.5M - 1.5.0 0da6a335fe13 97e01d7c5fff 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:node-exporter.b smithi105 *:9100 running (24m) 2m ago 33m 15.8M - 1.5.0 0da6a335fe13 5981b9513cda 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:osd.0 smithi081 running (7m) 2m ago 36m 54.6M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 2ecc2e37fa20 2024-09-17T01:58:32.119 INFO:teuthology.orchestra.run.smithi081.stdout:osd.1 smithi081 running (7m) 2m ago 36m 67.1M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 89df412a2a00 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.2 smithi081 running (6m) 2m ago 35m 55.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 06d00d250229 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.3 smithi081 running (5m) 2m ago 35m 52.7M 2992M 19.1.1-330-gc2ddf972 ebb0615149de 4b9d7ce7d0c4 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.4 smithi105 running (4m) 2m ago 35m 52.4M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 6e076440683c 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.5 smithi105 running (4m) 2m ago 34m 48.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 53156ce0ece8 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.6 smithi105 running (3m) 2m ago 34m 47.3M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 7eb8f16b0286 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:osd.7 smithi105 running (3m) 2m ago 33m 55.5M 3248M 19.1.1-330-gc2ddf972 ebb0615149de 33ea52f4564a 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:prometheus.a smithi105 *:9095 running (11m) 2m ago 33m 68.0M - 2.43.0 a07b618ecd1d 553c2d0bc4eb 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi081.fnhdrk smithi081 *:8000 running (2m) 2m ago 31m 76.0M - 19.1.1-330-gc2ddf972 ebb0615149de 0f90aadb4099 2024-09-17T01:58:32.120 INFO:teuthology.orchestra.run.smithi081.stdout:rgw.foo.smithi105.ttsowb smithi105 *:8000 running (2m) 2m ago 31m 77.0M - 19.1.1-330-gc2ddf972 ebb0615149de bc82a4219daa 2024-09-17T01:58:32.745 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:58:33.053 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:32 smithi081 ceph-mon[77854]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:33.053 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:32 smithi081 ceph-mon[80183]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:33.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:32 smithi105 ceph-mon[63427]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:33.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:33 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:33] "GET /metrics HTTP/1.1" 200 38434 "" "Prometheus/2.43.0" 2024-09-17T01:58:34.199 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:33 smithi081 ceph-mon[77854]: from='client.64846 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:34.199 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:33 smithi081 ceph-mon[80183]: from='client.64846 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:34.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:33 smithi105 ceph-mon[63427]: from='client.64846 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:34.840 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:58:34.840 INFO:teuthology.orchestra.run.smithi081.stdout: "mon": { 2024-09-17T01:58:34.840 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:58:34.840 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:58:34.840 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr": { 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "osd": { 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw": { 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: }, 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "overall": { 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 15 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout: } 2024-09-17T01:58:34.841 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:58:35.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:34 smithi081 ceph-mon[77854]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:35.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:34 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/4196479073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:35.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:34 smithi081 ceph-mon[80183]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:35.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:34 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/4196479073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:34 smithi105 ceph-mon[63427]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:35.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:34 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/4196479073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:35.500 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:58:37.016 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:36 smithi081 ceph-mon[77854]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:37.016 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:36 smithi081 ceph-mon[80183]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:37.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:36 smithi105 ceph-mon[63427]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "in_progress": true, 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "services_complete": [ 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "osd", 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "mgr", 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "rgw", 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: "mon" 2024-09-17T01:58:37.441 INFO:teuthology.orchestra.run.smithi081.stdout: ], 2024-09-17T01:58:37.442 INFO:teuthology.orchestra.run.smithi081.stdout: "progress": "15/21 daemons upgraded", 2024-09-17T01:58:37.442 INFO:teuthology.orchestra.run.smithi081.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed.", 2024-09-17T01:58:37.442 INFO:teuthology.orchestra.run.smithi081.stdout: "is_paused": true 2024-09-17T01:58:37.442 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:58:38.100 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:58:39.297 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:38 smithi081 ceph-mon[77854]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:39.297 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:38 smithi081 ceph-mon[77854]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:39.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:38 smithi081 ceph-mon[80183]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:39.297 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:38 smithi081 ceph-mon[80183]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:39.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:38 smithi105 ceph-mon[63427]: from='client.54882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:39.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:38 smithi105 ceph-mon[63427]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout: daemon iscsi.foo.smithi081.umxnyg on smithi081 is in error state 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi081.umxnyg on host smithi081 failed. 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout: Upgrade daemon: iscsi.foo.smithi081.umxnyg: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi081.umxnyg ... 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:Creating ceph-iscsi config... 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/iscsi-gateway.cfg 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:Write file: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/iscsi.foo.smithi081.umxnyg/tcmu-runner-entrypoint.sh 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2dded6f264\x2d7492\x2d11ef\x2dbceb\x2dc7b262605968.slice/ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service 2024-09-17T01:58:40.108 INFO:teuthology.orchestra.run.smithi081.stdout:Non-zero exit code 1 from systemctl start ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout:systemctl: stderr Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout:systemctl: stderr See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout:Traceback (most recent call last): 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: return _run_code(code, main_globals, None, 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: exec(code, run_globals) 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5579, in 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 5567, in main 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-09-17T01:58:40.109 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-09-17T01:58:40.110 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1088, in deploy_daemon 2024-09-17T01:58:40.110 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/__main__.py", line 1224, in deploy_daemon_units 2024-09-17T01:58:40.110 INFO:teuthology.orchestra.run.smithi081.stdout: File "/tmp/tmpnroy8d0w.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-09-17T01:58:40.110 INFO:teuthology.orchestra.run.smithi081.stdout:RuntimeError: Failed command: systemctl start ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg: Job for ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service failed because the control process exited with error code. 2024-09-17T01:58:40.110 INFO:teuthology.orchestra.run.smithi081.stdout:See "systemctl status ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" and "journalctl -xeu ceph-ded6f264-7492-11ef-bceb-c7b262605968@iscsi.foo.smithi081.umxnyg.service" for details. 2024-09-17T01:58:40.704 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T01:58:41.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:40 smithi105 ceph-mon[63427]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:41.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:40 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:41.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:40 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1494956625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[77854]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1494956625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[80183]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:41.232 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:40 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1494956625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:58:42.804 INFO:teuthology.orchestra.run.smithi081.stdout:true 2024-09-17T01:58:43.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:42 smithi081 ceph-mon[77854]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:43.109 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:42 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/1361044989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:43.109 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:42 smithi081 ceph-mon[80183]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:43.109 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:42 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/1361044989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:42 smithi105 ceph-mon[63427]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:43.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:42 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/1361044989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:43.434 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T01:58:43.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:43 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:43] "GET /metrics HTTP/1.1" 200 38434 "" "Prometheus/2.43.0" 2024-09-17T01:58:43.980 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:43 smithi081 ceph-mon[77854]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:43.980 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:43 smithi081 ceph-mon[80183]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:44.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:43 smithi105 ceph-mon[63427]: from='mgr.54108 172.21.15.81:0/3366695993' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:58:45.015 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:44 smithi081 ceph-mon[77854]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:45.016 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:44 smithi081 ceph-mon[80183]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:45.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:44 smithi105 ceph-mon[63427]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:45.529 INFO:teuthology.orchestra.run.smithi081.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:58:46.174 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:45 smithi081 ceph-mon[77854]: from='client.? 172.21.15.81:0/2755561133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:46.174 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:45 smithi081 ceph-mon[80183]: from='client.? 172.21.15.81:0/2755561133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:46.198 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-09-17T01:58:46.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:45 smithi105 ceph-mon[63427]: from='client.? 172.21.15.81:0/2755561133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:58:47.180 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:46 smithi081 ceph-mon[77854]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:47.181 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:46 smithi081 ceph-mon[80183]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:47.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:46 smithi105 ceph-mon[63427]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:48.198 INFO:teuthology.orchestra.run.smithi081.stdout:osd 8 2m ago - 2024-09-17T01:58:48.821 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:58:48.835 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T01:58:48.835 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-09-17T01:58:49.134 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:48 smithi081 ceph-mon[77854]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:49.134 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:48 smithi081 ceph-mon[80183]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:49.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:48 smithi105 ceph-mon[63427]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:50.170 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:49 smithi081 ceph-mon[77854]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:50.170 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:49 smithi081 ceph-mon[80183]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:50.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:49 smithi105 ceph-mon[63427]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout:{ 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "image": "quay.io/ceph/ceph", 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "registry": "quay.io", 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "bare_image": "ceph/ceph", 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "versions": [ 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "19.1.1", 2024-09-17T01:58:51.089 INFO:teuthology.orchestra.run.smithi081.stdout: "19.1.0" 2024-09-17T01:58:51.090 INFO:teuthology.orchestra.run.smithi081.stdout: ] 2024-09-17T01:58:51.090 INFO:teuthology.orchestra.run.smithi081.stdout:} 2024-09-17T01:58:51.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:50 smithi105 ceph-mon[63427]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:51.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:50 smithi105 ceph-mon[63427]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:51.410 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:50 smithi081 ceph-mon[77854]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:51.410 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:50 smithi081 ceph-mon[77854]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:51.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:50 smithi081 ceph-mon[80183]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:51.410 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:50 smithi081 ceph-mon[80183]: from='client.64771 -' entity='client.iscsi.foo.smithi081.umxnyg' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T01:58:51.708 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-09-17T01:58:52.305 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:51 smithi081 ceph-mon[77854]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:52.305 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:51 smithi081 ceph-mon[80183]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:52.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:51 smithi105 ceph-mon[63427]: from='client.64888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:53.244 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:52 smithi081 ceph-mon[77854]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:53.245 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:52 smithi081 ceph-mon[80183]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:53.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:52 smithi105 ceph-mon[63427]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:53.680 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:53 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: ::ffff:172.21.15.105 - - [17/Sep/2024:01:58:53] "GET /metrics HTTP/1.1" 200 38436 "" "Prometheus/2.43.0" 2024-09-17T01:58:53.874 INFO:teuthology.orchestra.run.smithi081.stdout: "16.2.0", 2024-09-17T01:58:54.463 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ded6f264-7492-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-09-17T01:58:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:54 smithi105 ceph-mon[63427]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:55.385 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:54 smithi105 ceph-mon[63427]: from='client.54918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:54 smithi081 ceph-mon[77854]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:55.430 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:54 smithi081 ceph-mon[77854]: from='client.54918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:55.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:54 smithi081 ceph-mon[80183]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T01:58:55.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:54 smithi081 ceph-mon[80183]: from='client.54918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:58:56.671 INFO:teuthology.orchestra.run.smithi081.stdout: "v16.2.2", 2024-09-17T01:58:56.671 INFO:teuthology.orchestra.run.smithi081.stdout: "v16.2.2-20210505", 2024-09-17T01:58:57.263 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:56 smithi081 ceph-mon[77854]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:57.263 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:56 smithi081 ceph-mon[80183]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:57.288 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T01:58:57.302 INFO:tasks.cephadm:Teardown begin 2024-09-17T01:58:57.303 DEBUG:teuthology.orchestra.run.smithi081:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:58:57.332 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:58:57.360 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:58:56 smithi105 ceph-mon[63427]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T01:58:57.366 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T01:58:57.366 DEBUG:teuthology.orchestra.run.smithi081:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T01:58:57.388 DEBUG:teuthology.orchestra.run.smithi105:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T01:58:57.424 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T01:58:57.424 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-17T01:58:57.425 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a 2024-09-17T01:58:57.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:57 smithi081 systemd[1]: Stopping Ceph mon.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:58:57.989 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:57] ENGINE Bus STOPPING 2024-09-17T01:58:57.989 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[77850]: 2024-09-17T01:58:57.726+0000 7f184f4b8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:58:57.989 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a[77850]: 2024-09-17T01:58:57.726+0000 7f184f4b8640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-17T01:58:57.989 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:57 smithi081 podman[121003]: 2024-09-17 01:58:57.880220682 +0000 UTC m=+0.259616731 container died 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T01:58:58.299 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-17T01:58:58.299 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:57] ENGINE Bus STOPPED 2024-09-17T01:58:58.300 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:57 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:57] ENGINE Bus STARTING 2024-09-17T01:58:58.300 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:58] ENGINE Serving on http://:::9283 2024-09-17T01:58:58.300 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:58:58 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: [17/Sep/2024:01:58:58] ENGINE Bus STARTED 2024-09-17T01:58:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:58 smithi081 podman[121003]: 2024-09-17 01:58:58.302015549 +0000 UTC m=+0.681411601 container cleanup 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:58:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:58 smithi081 bash[121003]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a 2024-09-17T01:58:58.680 INFO:journalctl@ceph.mon.a.smithi081.stdout:Sep 17 01:58:58 smithi081 podman[121016]: 2024-09-17 01:58:58.478582157 +0000 UTC m=+0.598851367 container remove 1dd8805b728c8c6e9d459d42916841553bb28627297f1b73ef97b4462ef523b0 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS) 2024-09-17T01:58:58.722 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.a.service' 2024-09-17T01:58:58.767 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:58:58.767 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-17T01:58:58.767 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-17T01:58:58.767 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.c 2024-09-17T01:58:59.112 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:58 smithi081 systemd[1]: Stopping Ceph mon.c for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:58:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c[80179]: 2024-09-17T01:58:59.113+0000 7f02b4e6e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:58:59.430 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:59 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c[80179]: 2024-09-17T01:58:59.113+0000 7f02b4e6e640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-09-17T01:58:59.431 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:59 smithi081 podman[121138]: 2024-09-17 01:58:59.277993374 +0000 UTC m=+0.281371517 container died c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:58:59.813 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:59 smithi081 podman[121138]: 2024-09-17 01:58:59.58936087 +0000 UTC m=+0.592739002 container cleanup c257f6ba01211667ee5327396eeea6432162026a5d6821a1ae21efd7d0e1e2da (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-17T01:58:59.813 INFO:journalctl@ceph.mon.c.smithi081.stdout:Sep 17 01:58:59 smithi081 bash[121138]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-c 2024-09-17T01:58:59.937 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.c.service' 2024-09-17T01:58:59.972 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:58:59.972 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-17T01:58:59.972 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-17T01:58:59.972 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.b 2024-09-17T01:59:00.291 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:00 smithi105 systemd[1]: Stopping Ceph mon.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:00.635 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b[63423]: 2024-09-17T01:59:00.291+0000 7fdeb9d61640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b[63423]: 2024-09-17T01:59:00.291+0000 7fdeb9d61640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-09-17T01:59:00.636 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:00 smithi105 podman[96467]: 2024-09-17 01:59:00.538692714 +0000 UTC m=+0.363581792 container died 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:01.135 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:01 smithi105 podman[96467]: 2024-09-17 01:59:01.060732792 +0000 UTC m=+0.885621912 container cleanup 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:01.136 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:01 smithi105 bash[96467]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b 2024-09-17T01:59:01.613 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 01:59:01 smithi105 podman[96479]: 2024-09-17 01:59:01.362777529 +0000 UTC m=+0.820836262 container remove 02f027eec663b9f9305a671240910d47739a1695811f47846beccfd4fc2fc7ec (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T01:59:01.632 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mon.b.service' 2024-09-17T01:59:01.674 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:01.674 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-17T01:59:01.674 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-17T01:59:01.674 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y 2024-09-17T01:59:01.974 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:01 smithi081 systemd[1]: Stopping Ceph mgr.y for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:02.283 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:01 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:59:01.976+0000 7f46f97e3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:02.284 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:01 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y[60917]: 2024-09-17T01:59:01.976+0000 7f46f97e3640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-17T01:59:02.284 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:02 smithi081 podman[121257]: 2024-09-17 01:59:02.01789005 +0000 UTC m=+0.148239870 container died a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:59:02.669 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:02 smithi081 podman[121257]: 2024-09-17 01:59:02.360051416 +0000 UTC m=+0.490401223 container cleanup a7ee288e6ac2639a253a6c9b13bbb0c0c72c48962c35f9e9cdafb92a17142ab7 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T01:59:02.669 INFO:journalctl@ceph.mgr.y.smithi081.stdout:Sep 17 01:59:02 smithi081 bash[121257]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-y 2024-09-17T01:59:02.836 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-17T01:59:02.870 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:02.870 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-17T01:59:02.871 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-17T01:59:02.871 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x 2024-09-17T01:59:03.195 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:02 smithi105 systemd[1]: Stopping Ceph mgr.x for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:03.463 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:03 smithi105 podman[96590]: 2024-09-17 01:59:03.196941253 +0000 UTC m=+0.131289229 container died a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:03.840 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:03 smithi105 podman[96590]: 2024-09-17 01:59:03.624085317 +0000 UTC m=+0.558433284 container cleanup a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:03.841 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:03 smithi105 bash[96590]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x 2024-09-17T01:59:04.099 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:03 smithi105 podman[96602]: 2024-09-17 01:59:03.842818351 +0000 UTC m=+0.645761632 container remove a4d2bd4a6303387f46fae3c17d976be720b189b69260353f544efd367faa9e15 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T01:59:04.099 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 01:59:03 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-17T01:59:04.103 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-17T01:59:04.137 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:04.138 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-17T01:59:04.138 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T01:59:04.138 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.0 2024-09-17T01:59:04.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:04 smithi081 systemd[1]: Stopping Ceph osd.0 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:04.431 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:04.265Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:04.431 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:04.266Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:04.930 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:59:04.449+0000 7f958fa34640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:04.930 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:59:04.449+0000 7f958fa34640 -1 osd.0 159 *** Got signal Terminated *** 2024-09-17T01:59:04.931 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0[87088]: 2024-09-17T01:59:04.449+0000 7f958fa34640 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:10.140 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:09 smithi081 podman[121380]: 2024-09-17 01:59:09.872591732 +0000 UTC m=+5.543790003 container died 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T01:59:10.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:10 smithi081 podman[121380]: 2024-09-17 01:59:10.233036463 +0000 UTC m=+5.904234733 container cleanup 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:59:10.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:10 smithi081 bash[121380]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0 2024-09-17T01:59:10.845 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:10 smithi081 podman[121396]: 2024-09-17 01:59:10.526080817 +0000 UTC m=+0.647837546 container remove 2ecc2e37fa20589da1307e2baed9cc86ad9a441abe0ca0d0ddccb6353aca2a73 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T01:59:11.102 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:10 smithi081 podman[121487]: 2024-09-17 01:59:10.750863132 +0000 UTC m=+0.015958622 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:11.102 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:10 smithi081 podman[121487]: 2024-09-17 01:59:10.868467357 +0000 UTC m=+0.133562838 container create 029b54978744e9617259846a74e7eb40d671f0cea8c1a2d81193187e2fcb6f94 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T01:59:11.102 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:11 smithi081 podman[121487]: 2024-09-17 01:59:11.104581081 +0000 UTC m=+0.369676560 container init 029b54978744e9617259846a74e7eb40d671f0cea8c1a2d81193187e2fcb6f94 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T01:59:11.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:11 smithi081 podman[121487]: 2024-09-17 01:59:11.107927691 +0000 UTC m=+0.373023171 container start 029b54978744e9617259846a74e7eb40d671f0cea8c1a2d81193187e2fcb6f94 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:59:11.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:11 smithi081 podman[121487]: 2024-09-17 01:59:11.204618784 +0000 UTC m=+0.469714264 container attach 029b54978744e9617259846a74e7eb40d671f0cea8c1a2d81193187e2fcb6f94 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T01:59:11.430 INFO:journalctl@ceph.osd.0.smithi081.stdout:Sep 17 01:59:11 smithi081 podman[121487]: 2024-09-17 01:59:11.300210225 +0000 UTC m=+0.565305717 container died 029b54978744e9617259846a74e7eb40d671f0cea8c1a2d81193187e2fcb6f94 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:11.934 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T01:59:11.973 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:11.974 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T01:59:11.974 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T01:59:11.974 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.1 2024-09-17T01:59:12.180 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:12 smithi081 systemd[1]: Stopping Ceph osd.1 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:12.680 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:59:12.344+0000 7f5bd885b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:12.680 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:59:12.344+0000 7f5bd885b640 -1 osd.1 159 *** Got signal Terminated *** 2024-09-17T01:59:12.680 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:12 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1[92190]: 2024-09-17T01:59:12.344+0000 7f5bd885b640 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:14.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:14.266Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:14.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:14.266Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:14.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:14 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:14.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:17.926 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:17 smithi081 podman[121593]: 2024-09-17 01:59:17.672966713 +0000 UTC m=+5.432965547 container died 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:18.180 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121593]: 2024-09-17 01:59:18.019635118 +0000 UTC m=+5.779633942 container cleanup 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:18.181 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 bash[121593]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1 2024-09-17T01:59:18.638 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121619]: 2024-09-17 01:59:18.313059105 +0000 UTC m=+0.642301011 container remove 89df412a2a00af7eb064f4e53e80f63d8a4438a635a57e27eadc115b453bb9e9 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:18.906 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121712]: 2024-09-17 01:59:18.543034867 +0000 UTC m=+0.015717746 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:18.906 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121712]: 2024-09-17 01:59:18.680663168 +0000 UTC m=+0.153346038 container create 51648684ad97c2e85d6e028412db92d5340f78b3a727ceab2631577ed2d63989 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:59:18.906 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121712]: 2024-09-17 01:59:18.908197889 +0000 UTC m=+0.380880761 container init 51648684ad97c2e85d6e028412db92d5340f78b3a727ceab2631577ed2d63989 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T01:59:19.180 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121712]: 2024-09-17 01:59:18.911755966 +0000 UTC m=+0.384438835 container start 51648684ad97c2e85d6e028412db92d5340f78b3a727ceab2631577ed2d63989 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-17T01:59:19.181 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:18 smithi081 podman[121712]: 2024-09-17 01:59:18.999734239 +0000 UTC m=+0.472417127 container attach 51648684ad97c2e85d6e028412db92d5340f78b3a727ceab2631577ed2d63989 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T01:59:19.181 INFO:journalctl@ceph.osd.1.smithi081.stdout:Sep 17 01:59:19 smithi081 podman[121712]: 2024-09-17 01:59:19.098618921 +0000 UTC m=+0.571301799 container died 51648684ad97c2e85d6e028412db92d5340f78b3a727ceab2631577ed2d63989 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux ) 2024-09-17T01:59:19.685 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T01:59:19.724 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:19.728 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T01:59:19.728 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T01:59:19.728 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.2 2024-09-17T01:59:19.930 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:19 smithi081 systemd[1]: Stopping Ceph osd.2 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:20.430 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:59:20.055+0000 7fb94ac85640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:20.430 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:59:20.055+0000 7fb94ac85640 -1 osd.2 159 *** Got signal Terminated *** 2024-09-17T01:59:20.430 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:20 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2[98447]: 2024-09-17T01:59:20.055+0000 7fb94ac85640 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:24.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:24.266Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:24.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:24.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:24.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:24 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:24.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:25.846 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:25 smithi081 podman[121817]: 2024-09-17 01:59:25.577460213 +0000 UTC m=+5.624219271 container died 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:26.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:25 smithi081 podman[121817]: 2024-09-17 01:59:25.922392208 +0000 UTC m=+5.969151253 container cleanup 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T01:59:26.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:25 smithi081 bash[121817]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2 2024-09-17T01:59:26.538 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121845]: 2024-09-17 01:59:26.207495822 +0000 UTC m=+0.630836513 container remove 06d00d25022952dabb0ff37033e39eefbe4c7daca4be912cf27ff98cdf5d6455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T01:59:26.808 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.541031792 +0000 UTC m=+0.108093075 container create 9f1f521f52a07d5d44661b950ce426d58ab4a6ba4372428697b574f96fc1a24e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T01:59:26.808 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.448742401 +0000 UTC m=+0.015803692 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:26.808 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.726785613 +0000 UTC m=+0.293846910 container init 9f1f521f52a07d5d44661b950ce426d58ab4a6ba4372428697b574f96fc1a24e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T01:59:26.808 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.730128151 +0000 UTC m=+0.297189441 container start 9f1f521f52a07d5d44661b950ce426d58ab4a6ba4372428697b574f96fc1a24e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T01:59:26.808 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.810149699 +0000 UTC m=+0.377210994 container attach 9f1f521f52a07d5d44661b950ce426d58ab4a6ba4372428697b574f96fc1a24e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T01:59:27.180 INFO:journalctl@ceph.osd.2.smithi081.stdout:Sep 17 01:59:26 smithi081 podman[121934]: 2024-09-17 01:59:26.922006046 +0000 UTC m=+0.489067333 container died 9f1f521f52a07d5d44661b950ce426d58ab4a6ba4372428697b574f96fc1a24e (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T01:59:27.575 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T01:59:27.616 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:27.617 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T01:59:27.617 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T01:59:27.617 DEBUG:teuthology.orchestra.run.smithi081:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.3 2024-09-17T01:59:27.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:27 smithi081 systemd[1]: Stopping Ceph osd.3 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:28.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:27.933+0000 7f893460f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:28.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:27.933+0000 7f893460f640 -1 osd.3 159 *** Got signal Terminated *** 2024-09-17T01:59:28.430 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:27 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:27.933+0000 7f893460f640 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:31.180 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:30 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:30.874+0000 7f8930c25640 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:09.071926+0000 front 2024-09-17T01:59:09.071945+0000 (oldest deadline 2024-09-17T01:59:30.772046+0000) 2024-09-17T01:59:31.556 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:31.153+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:31.885 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:31 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:31.715+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:32.180 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:31 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:31.875+0000 7f8930c25640 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:09.071926+0000 front 2024-09-17T01:59:09.071945+0000 (oldest deadline 2024-09-17T01:59:30.772046+0000) 2024-09-17T01:59:32.385 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:32.059+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:32.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:32.130+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:32.730 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:32.431+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:33.038 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:33.040+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:33.038 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:32 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:32.737+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:33.180 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:32 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3[104935]: 2024-09-17T01:59:32.909+0000 7f8930c25640 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:09.071926+0000 front 2024-09-17T01:59:09.071945+0000 (oldest deadline 2024-09-17T01:59:30.772046+0000) 2024-09-17T01:59:33.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:33.175+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:33.648 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:33 smithi081 podman[122035]: 2024-09-17 01:59:33.356806596 +0000 UTC m=+5.516727350 container died 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T01:59:33.704 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:33.412+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:33.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:33 smithi081 podman[122035]: 2024-09-17 01:59:33.741317571 +0000 UTC m=+5.901238315 container cleanup 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:33.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:33 smithi081 bash[122035]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3 2024-09-17T01:59:34.036 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:33 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:33.705+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:34.265 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:33 smithi081 podman[122051]: 2024-09-17 01:59:33.98420322 +0000 UTC m=+0.621506362 container remove 4b9d7ce7d0c4171c07f33af8361ec80d574d1aebd017b28d67336d0f3b5632af (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T01:59:34.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:34.144+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:34.386 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:34.038+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:34.518 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.284928311 +0000 UTC m=+0.084030497 container create 70eae105aa5ce54cbc7ef3924dab717ca994fc99309d3ed23db9df057da17688 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:34.518 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.216687384 +0000 UTC m=+0.015789537 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:34.518 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.520989557 +0000 UTC m=+0.320091698 container init 70eae105aa5ce54cbc7ef3924dab717ca994fc99309d3ed23db9df057da17688 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:34.519 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:34.267Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:34.519 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:34.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:34.519 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:34 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:34.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:34.672 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:34.408+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:34.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.524832214 +0000 UTC m=+0.323934357 container start 70eae105aa5ce54cbc7ef3924dab717ca994fc99309d3ed23db9df057da17688 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T01:59:34.930 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.604198496 +0000 UTC m=+0.403300655 container attach 70eae105aa5ce54cbc7ef3924dab717ca994fc99309d3ed23db9df057da17688 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T01:59:34.931 INFO:journalctl@ceph.osd.3.smithi081.stdout:Sep 17 01:59:34 smithi081 podman[122137]: 2024-09-17 01:59:34.710010349 +0000 UTC m=+0.509112503 container died 70eae105aa5ce54cbc7ef3924dab717ca994fc99309d3ed23db9df057da17688 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:34.994 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:34.674+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:35.309 DEBUG:teuthology.orchestra.run.smithi081:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T01:59:35.352 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:35.352 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T01:59:35.352 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T01:59:35.352 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4 2024-09-17T01:59:35.357 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:35.115+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:35.357 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:34 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:34.997+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:35.635 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:35 smithi105 systemd[1]: Stopping Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:35.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:35.358+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:36.016 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:35.645+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:36.016 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:35.751+0000 7f1fbc0b1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:36.016 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:35.751+0000 7f1fbc0b1640 -1 osd.4 159 *** Got signal Terminated *** 2024-09-17T01:59:36.016 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:35 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:35.751+0000 7f1fbc0b1640 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:36.016 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:36 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:36.016+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:36.327 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:36 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:36.160+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:36.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:36 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:36.329+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:36.993 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:36 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:36.685+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:37.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:37.184+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:37.279 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:36 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:36.995+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:37.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:37.280+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:38.005 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:37 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:37.719+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:38.301 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:38.181+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:38.301 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:38.006+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:38.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:38.302+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:38.992 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:38.741+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:39.311 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:38.993+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:39.311 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:39.150+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:39.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:39.312+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:40.001 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:39.720+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:40.002 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:39 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:39.720+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:40.287 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:40.198+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:40.287 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:40.002+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:08.604611+0000 front 2024-09-17T01:59:08.604521+0000 (oldest deadline 2024-09-17T01:59:31.504458+0000) 2024-09-17T01:59:40.287 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4[73328]: 2024-09-17T01:59:40.002+0000 7f1fb86c7640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:14.405651+0000 front 2024-09-17T01:59:14.405560+0000 (oldest deadline 2024-09-17T01:59:39.105541+0000) 2024-09-17T01:59:40.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:40.287+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:40.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:40.287+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:41.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:40.692+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:41.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:40 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:40.692+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:41.427 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:41 smithi105 podman[96726]: 2024-09-17 01:59:41.176384299 +0000 UTC m=+5.617935606 container died 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:41.427 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:41.292+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:41.427 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:41.292+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:41.427 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:41.238+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:41.428 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:41.238+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:41.733 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:41 smithi105 podman[96726]: 2024-09-17 01:59:41.63586989 +0000 UTC m=+6.077421194 container cleanup 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-c2ddf97, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:41.733 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:41 smithi105 bash[96726]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4 2024-09-17T01:59:42.135 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:41 smithi105 podman[96741]: 2024-09-17 01:59:41.92958785 +0000 UTC m=+0.746660202 container remove 6e076440683c6b26503193bb90329ae637a704db45de5a6af38d88ecb2906727 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:42.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:41.734+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:42.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:41 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:41.734+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:42.505 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.255327373 +0000 UTC m=+0.112527288 container create 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-17T01:59:42.505 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.158657345 +0000 UTC m=+0.015857259 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:42.505 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:42.330+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:42.506 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:42.330+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:42.506 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:42.206+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:42.506 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:42.206+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:42.797 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:42.719+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:42.797 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:42 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:42.719+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:42.798 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.507900036 +0000 UTC m=+0.365099952 container init 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T01:59:42.798 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.511478529 +0000 UTC m=+0.368678450 container start 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T01:59:42.798 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.591087741 +0000 UTC m=+0.448287674 container attach 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:59:42.798 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:42 smithi105 podman[96831]: 2024-09-17 01:59:42.698567252 +0000 UTC m=+0.555767175 container died 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731) 2024-09-17T01:59:43.353 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T01:59:43.417 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:43.157+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:43.157+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:43 smithi105 podman[96854]: 2024-09-17 01:59:43.333326905 +0000 UTC m=+0.629100243 container remove 4bc03d2ce48516c32be2dc1e5ea8708568d9ac2efb47ecb154169e26609ac492 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-c2ddf97, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:43 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:43 smithi105 systemd[1]: Stopped Ceph osd.4 for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 01:59:43 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.4.service: Consumed 5.366s CPU time. 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:43.310+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:43.418 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:43.310+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:43.433 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:43.434 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T01:59:43.434 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T01:59:43.434 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.5 2024-09-17T01:59:43.737 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 systemd[1]: Stopping Ceph osd.5 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:44.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:43.737+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:44.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:43.737+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:44.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:43.822+0000 7fd3e0939640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:44.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:43.822+0000 7fd3e0939640 -1 osd.5 159 *** Got signal Terminated *** 2024-09-17T01:59:44.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:43 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:43.822+0000 7fd3e0939640 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:44.553 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:44.288+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:44.554 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:44.288+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:44.554 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:44.164+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:44.554 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:44.164+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:44.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:44.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:44.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:44.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:44.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:44 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:44.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:44.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:44.765+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:44.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:44 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:44.765+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:45.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:45.254+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:45.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:45.254+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:45.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:45.155+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:45.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:45.155+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:46.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:45.792+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:46.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:45 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:45.792+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:46.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:46.209+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:46.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:46.209+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:46.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:46.209+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:46.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:46.155+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:46.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:46.155+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:47.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:46.825+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:47.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:46 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:46.825+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:47.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:47.160+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:47.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:47.160+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:47.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:47.160+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:47.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:47.163+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:47.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:47.163+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:48.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:47.871+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:06.684514+0000 front 2024-09-17T01:59:06.684504+0000 (oldest deadline 2024-09-17T01:59:30.784372+0000) 2024-09-17T01:59:48.135 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:47.871+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.085319+0000 front 2024-09-17T01:59:16.085401+0000 (oldest deadline 2024-09-17T01:59:38.985340+0000) 2024-09-17T01:59:48.136 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:47 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5[78351]: 2024-09-17T01:59:47.871+0000 7fd3dcf4f640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:21.786709+0000 front 2024-09-17T01:59:21.786812+0000 (oldest deadline 2024-09-17T01:59:47.086631+0000) 2024-09-17T01:59:48.635 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:48.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:48.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:48.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:48.636 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:48.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:48.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:48.141+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:48.636 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:48 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:48.141+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:49.415 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:49 smithi105 podman[96935]: 2024-09-17 01:59:49.138882846 +0000 UTC m=+5.445031190 container died 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux ) 2024-09-17T01:59:49.415 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:49.140+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:49.415 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:49.140+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:49.415 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:49.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:49.415 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:49.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:49.416 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:49 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:49.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:49.885 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:49 smithi105 podman[96935]: 2024-09-17 01:59:49.590164516 +0000 UTC m=+5.896312814 container cleanup 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97) 2024-09-17T01:59:49.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:49 smithi105 bash[96935]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5 2024-09-17T01:59:50.263 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:49 smithi105 podman[96960]: 2024-09-17 01:59:49.925207542 +0000 UTC m=+0.784032875 container remove 53156ce0ece8b2c287d6428a4ec2d80d4e807d843c33da17d1859432162ad816 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T01:59:50.263 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:50.128+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:50.263 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:50.128+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:50.263 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:50.128+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:50.264 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:50.111+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:50.264 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:50.111+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:50.264 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:50 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:50.111+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:50.584 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.168312762 +0000 UTC m=+0.015765080 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:50.584 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.284383023 +0000 UTC m=+0.131835331 container create fe69a925d8df3d89ce78d142996ed21210ef952dbba7b58002617a1e778fa821 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:50.585 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.503828231 +0000 UTC m=+0.351280547 container init fe69a925d8df3d89ce78d142996ed21210ef952dbba7b58002617a1e778fa821 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T01:59:50.585 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.50701578 +0000 UTC m=+0.354468098 container start fe69a925d8df3d89ce78d142996ed21210ef952dbba7b58002617a1e778fa821 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:59:50.885 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.586980327 +0000 UTC m=+0.434432640 container attach fe69a925d8df3d89ce78d142996ed21210ef952dbba7b58002617a1e778fa821 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731) 2024-09-17T01:59:50.886 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 01:59:50 smithi105 podman[97047]: 2024-09-17 01:59:50.700053816 +0000 UTC m=+0.547506127 container died fe69a925d8df3d89ce78d142996ed21210ef952dbba7b58002617a1e778fa821 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T01:59:51.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.138+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:51.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.138+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:51.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.138+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:51.387 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:51.100+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:51.387 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:51.100+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:51.387 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:51.100+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:51.419 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T01:59:51.462 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:51.462 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T01:59:51.463 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T01:59:51.463 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.6 2024-09-17T01:59:51.825 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 systemd[1]: Stopping Ceph osd.6 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T01:59:51.825 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.827+0000 7fd2363bb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:59:51.825 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.827+0000 7fd2363bb640 -1 osd.6 159 *** Got signal Terminated *** 2024-09-17T01:59:52.094 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:51 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:51.827+0000 7fd2363bb640 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:52.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:52.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:52.152+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:52.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:52.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:52.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:52 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:52.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:53.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:53.093+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:53.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:53.093+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:53.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:53.093+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:53.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:53.156+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:53.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:53.156+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:53.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:53 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:53.156+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:54.132+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:54.132+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:54.132+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:54.110+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:54.110+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:54.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:54 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:54.110+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:54.591 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:54.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:54.591 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:54.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T01:59:54.592 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 01:59:54 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T01:59:54.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:55.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:55.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:55.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:55.151+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:31.311433+0000 front 2024-09-17T01:59:31.311379+0000 (oldest deadline 2024-09-17T01:59:54.211334+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:55.062+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:55.062+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:55.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:55 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:55.062+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:56.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:56.020+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:56.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:56.020+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:56.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:56.020+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:56.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:56.123+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:07.908707+0000 front 2024-09-17T01:59:07.908646+0000 (oldest deadline 2024-09-17T01:59:31.408493+0000) 2024-09-17T01:59:56.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:56.123+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:17.209961+0000 front 2024-09-17T01:59:17.210096+0000 (oldest deadline 2024-09-17T01:59:40.110002+0000) 2024-09-17T01:59:56.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:56.123+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:20.110438+0000 front 2024-09-17T01:59:20.110511+0000 (oldest deadline 2024-09-17T01:59:46.010409+0000) 2024-09-17T01:59:56.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:56 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6[82852]: 2024-09-17T01:59:56.123+0000 7fd2329d1640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:31.311433+0000 front 2024-09-17T01:59:31.311379+0000 (oldest deadline 2024-09-17T01:59:54.211334+0000) 2024-09-17T01:59:57.386 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:57 smithi105 podman[97151]: 2024-09-17 01:59:57.255592233 +0000 UTC m=+5.559827252 container died 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:57.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:57.068+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:57.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:57.068+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:57.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:57.068+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:57.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:57.068+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T01:59:57.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:57 smithi105 podman[97151]: 2024-09-17 01:59:57.72722512 +0000 UTC m=+6.031460174 container cleanup 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T01:59:57.885 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:57 smithi105 bash[97151]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6 2024-09-17T01:59:58.337 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:57 smithi105 podman[97168]: 2024-09-17 01:59:57.995459015 +0000 UTC m=+0.739953085 container remove 7eb8f16b0286eeb21e8b18f15c8c702e4c072af1528f9dbea9ce88ce4c5f5710 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T01:59:58.338 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:58.105+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:58.338 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:58.105+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:58.338 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:58.105+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:58.339 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:58 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:58.105+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T01:59:58.339 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:59:57.935Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.81:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.81:8765: connect: connection refused" 2024-09-17T01:59:58.339 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:59:57.939Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.81:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.81:8765: connect: connection refused" 2024-09-17T01:59:58.340 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:59:57.939Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.81:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.81:8765: connect: connection refused" 2024-09-17T01:59:58.340 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:59:57.939Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.81:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.81:8765: connect: connection refused" 2024-09-17T01:59:58.340 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 01:59:57 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T01:59:57.940Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.81:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.81:8765: connect: connection refused" 2024-09-17T01:59:58.605 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.242659316 +0000 UTC m=+0.016245782 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:59:58.605 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.366732616 +0000 UTC m=+0.140319075 container create f7dae5aa518ae06eca55ed1e040aabc9d0a46b572b6e590904a74ba6aab9151a (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T01:59:58.605 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.606978863 +0000 UTC m=+0.380565325 container init f7dae5aa518ae06eca55ed1e040aabc9d0a46b572b6e590904a74ba6aab9151a (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:59:58.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.61059424 +0000 UTC m=+0.384180700 container start f7dae5aa518ae06eca55ed1e040aabc9d0a46b572b6e590904a74ba6aab9151a (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:59:58.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.690560655 +0000 UTC m=+0.464147134 container attach f7dae5aa518ae06eca55ed1e040aabc9d0a46b572b6e590904a74ba6aab9151a (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T01:59:58.886 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 01:59:58 smithi105 podman[97257]: 2024-09-17 01:59:58.799788538 +0000 UTC m=+0.573375001 container died f7dae5aa518ae06eca55ed1e040aabc9d0a46b572b6e590904a74ba6aab9151a (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-c2ddf97) 2024-09-17T01:59:59.366 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.097+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T01:59:59.366 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.097+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T01:59:59.366 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.097+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T01:59:59.367 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.097+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T01:59:59.385 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T01:59:59.422 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:59:59.423 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T01:59:59.423 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T01:59:59.423 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.7 2024-09-17T01:59:59.635 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 systemd[1]: Stopping Ceph osd.7 for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.780+0000 7fbf2f540640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.780+0000 7fbf2f540640 -1 osd.7 159 *** Got signal Terminated *** 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 01:59:59 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T01:59:59.780+0000 7fbf2f540640 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:00.121+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:00.121+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:00.121+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T02:00:00.119 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:00 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:00.121+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T02:00:01.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:01 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:01.122+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T02:00:01.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:01 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:01.122+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T02:00:01.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:01 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:01.122+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T02:00:01.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:01 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:01.122+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T02:00:02.385 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:02.116+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T02:00:02.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:02.116+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T02:00:02.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:02.116+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T02:00:02.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:02 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:02.116+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T02:00:03.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:03.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T02:00:03.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:03.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T02:00:03.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:03.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T02:00:03.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:03.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T02:00:03.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:03 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:03.096+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6804 osd.4 since back 2024-09-17T01:59:39.376610+0000 front 2024-09-17T01:59:39.376682+0000 (oldest deadline 2024-09-17T02:00:02.276635+0000) 2024-09-17T02:00:04.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:04.046+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6806 osd.0 since back 2024-09-17T01:59:05.572353+0000 front 2024-09-17T01:59:05.572336+0000 (oldest deadline 2024-09-17T01:59:30.872176+0000) 2024-09-17T02:00:04.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:04.046+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6814 osd.1 since back 2024-09-17T01:59:16.173204+0000 front 2024-09-17T01:59:16.173224+0000 (oldest deadline 2024-09-17T01:59:40.273036+0000) 2024-09-17T02:00:04.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:04.046+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6822 osd.2 since back 2024-09-17T01:59:24.775197+0000 front 2024-09-17T01:59:24.775236+0000 (oldest deadline 2024-09-17T01:59:49.475032+0000) 2024-09-17T02:00:04.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:04.046+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.81:6830 osd.3 since back 2024-09-17T01:59:30.575892+0000 front 2024-09-17T01:59:30.575876+0000 (oldest deadline 2024-09-17T01:59:56.475816+0000) 2024-09-17T02:00:04.386 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:04 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7[87481]: 2024-09-17T02:00:04.046+0000 7fbf27b4e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6804 osd.4 since back 2024-09-17T01:59:39.376610+0000 front 2024-09-17T01:59:39.376682+0000 (oldest deadline 2024-09-17T02:00:02.276635+0000) 2024-09-17T02:00:04.680 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T02:00:04.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T02:00:04.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T02:00:04.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi081.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.81:8443: connect: connection refused" 2024-09-17T02:00:04.681 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:04 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T02:00:04.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi105.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-17T02:00:05.487 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:05 smithi105 podman[97360]: 2024-09-17 02:00:05.213376119 +0000 UTC m=+5.562602937 container died 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T02:00:05.845 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:05 smithi105 podman[97360]: 2024-09-17 02:00:05.563254668 +0000 UTC m=+5.912481486 container cleanup 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T02:00:05.845 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:05 smithi105 bash[97360]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7 2024-09-17T02:00:06.135 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:05 smithi105 podman[97388]: 2024-09-17 02:00:05.847900373 +0000 UTC m=+0.630911567 container remove 33ea52f4564aee50294db19147662fb47bb8ebcfc3c9d733f23fea0e6fc68455 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T02:00:06.440 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.09192253 +0000 UTC m=+0.015961855 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T02:00:06.440 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.198803921 +0000 UTC m=+0.122843286 container create fb88f2e83de33d27d749194b9404d114f4ef8510436b7501b436a91adf409589 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T02:00:06.440 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.443112206 +0000 UTC m=+0.367151538 container init fb88f2e83de33d27d749194b9404d114f4ef8510436b7501b436a91adf409589 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-09-17T02:00:06.885 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.446522303 +0000 UTC m=+0.370561636 container start fb88f2e83de33d27d749194b9404d114f4ef8510436b7501b436a91adf409589 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T02:00:06.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.526013591 +0000 UTC m=+0.450052938 container attach fb88f2e83de33d27d749194b9404d114f4ef8510436b7501b436a91adf409589 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-09-17T02:00:06.886 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 02:00:06 smithi105 podman[97477]: 2024-09-17 02:00:06.638927358 +0000 UTC m=+0.562966681 container died fb88f2e83de33d27d749194b9404d114f4ef8510436b7501b436a91adf409589 (image=quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T02:00:07.263 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T02:00:07.301 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T02:00:07.302 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T02:00:07.302 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-17T02:00:07.302 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a 2024-09-17T02:00:07.623 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 systemd[1]: Stopping Ceph prometheus.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.625Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-17T02:00:07.886 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-17T02:00:07.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T02:00:07.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-17T02:00:07.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a[59097]: ts=2024-09-17T02:00:07.626Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-17T02:00:07.887 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 podman[97580]: 2024-09-17 02:00:07.636245091 +0000 UTC m=+0.113126242 container died 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:08.265 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 podman[97580]: 2024-09-17 02:00:07.992231901 +0000 UTC m=+0.469113031 container cleanup 553c2d0bc4eb3c78b260da516b8519cc8da1fcf1b1f6b88fe05834383762e7aa (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:08.266 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 02:00:07 smithi105 bash[97580]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T02:00:08.419 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ded6f264-7492-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-17T02:00:08.454 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T02:00:08.454 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-17T02:00:08.454 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ded6f264-7492-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T02:00:10.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:09 smithi081 systemd[1]: Stopping Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:10.181 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 podman[122526]: 2024-09-17 02:00:10.152704635 +0000 UTC m=+0.088772057 container died 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:10.797 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 podman[122526]: 2024-09-17 02:00:10.491602423 +0000 UTC m=+0.427669834 container cleanup 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:10.797 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 bash[122526]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a 2024-09-17T02:00:10.797 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 podman[122539]: 2024-09-17 02:00:10.676130719 +0000 UTC m=+0.523654859 container remove 97e01d7c5fff9d31bdfccd46f9791bacac6200f3c895ab1694b812e5669cc5fb (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:10.797 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T02:00:11.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-17T02:00:11.180 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 systemd[1]: Stopped Ceph node-exporter.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T02:00:11.181 INFO:journalctl@ceph.node-exporter.a.smithi081.stdout:Sep 17 02:00:10 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 4.176s CPU time. 2024-09-17T02:00:11.181 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:10 smithi081 systemd[1]: Stopping Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:11.511 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a[57611]: ts=2024-09-17T02:00:11.220Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-17T02:00:11.511 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 podman[122648]: 2024-09-17 02:00:11.261961895 +0000 UTC m=+0.135639797 container died dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:11.886 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 podman[122648]: 2024-09-17 02:00:11.612558252 +0000 UTC m=+0.486236154 container cleanup dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:11.886 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 bash[122648]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a 2024-09-17T02:00:12.162 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 podman[122662]: 2024-09-17 02:00:11.888706139 +0000 UTC m=+0.621513158 container remove dc15deba3321cb5bd6829741f9e73eab96d42c5bb9428a5e92b7344f8605c52d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T02:00:12.163 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:11 smithi081 podman[122662]: 2024-09-17 02:00:11.973052784 +0000 UTC m=+0.705859757 volume remove dfd5d2ed5e046d7af1faa50307c1d23768169b84640a61be422da6fb8ecb4d97 2024-09-17T02:00:12.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:12 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T02:00:12.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:12 smithi081 systemd[1]: Stopped Ceph alertmanager.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T02:00:12.430 INFO:journalctl@ceph.alertmanager.a.smithi081.stdout:Sep 17 02:00:12 smithi081 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 2.143s CPU time. 2024-09-17T02:00:35.184 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ded6f264-7492-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T02:00:36.988 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:36 smithi105 systemd[1]: Stopping Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:37.259 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:36 smithi105 podman[97988]: 2024-09-17 02:00:36.989164821 +0000 UTC m=+0.125988444 container died 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T02:00:37.635 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 podman[97988]: 2024-09-17 02:00:37.4452091 +0000 UTC m=+0.582032722 container cleanup 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T02:00:37.636 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 bash[97988]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b 2024-09-17T02:00:38.121 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 podman[98000]: 2024-09-17 02:00:37.796439 +0000 UTC m=+0.808720597 container remove 5981b9513cda32cf9705309146f57b45dc5125f7d95a80046a072d0647bd7230 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T02:00:38.121 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-17T02:00:38.121 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-17T02:00:38.121 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 systemd[1]: Stopped Ceph node-exporter.b for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T02:00:38.121 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 02:00:37 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 4.193s CPU time. 2024-09-17T02:00:38.372 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 systemd[1]: Stopping Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968... 2024-09-17T02:00:38.623 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=server t=2024-09-17T02:00:38.374939221Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-17T02:00:38.623 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=ticker t=2024-09-17T02:00:38.375134268Z level=info msg=stopped last_tick=2024-09-17T02:00:30Z 2024-09-17T02:00:38.623 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a[61301]: logger=tracing t=2024-09-17T02:00:38.375143998Z level=info msg="Closing tracing" 2024-09-17T02:00:38.623 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 podman[98109]: 2024-09-17 02:00:38.38517064 +0000 UTC m=+0.122686582 container died 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T02:00:38.885 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 podman[98109]: 2024-09-17 02:00:38.69918646 +0000 UTC m=+0.436702390 container cleanup 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T02:00:38.886 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 bash[98109]: ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a 2024-09-17T02:00:38.886 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:38 smithi105 podman[98121]: 2024-09-17 02:00:38.850659957 +0000 UTC m=+0.466709989 container remove 16b0af893ffadf8fd65c21a4ed0215d0ab5cd2e6b9dd2edecb3ac5a2fddbae8a (image=quay.io/ceph/grafana:9.4.12, name=ceph-ded6f264-7492-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-17T02:00:39.385 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:39 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-17T02:00:39.385 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:39 smithi105 systemd[1]: Stopped Ceph grafana.a for ded6f264-7492-11ef-bceb-c7b262605968. 2024-09-17T02:00:39.386 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 02:00:39 smithi105 systemd[1]: ceph-ded6f264-7492-11ef-bceb-c7b262605968@grafana.a.service: Consumed 10.757s CPU time. 2024-09-17T02:00:50.790 DEBUG:teuthology.orchestra.run.smithi081:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T02:00:50.823 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T02:00:50.852 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T02:00:50.855 DEBUG:teuthology.misc:Transferring archived files from smithi081:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi081/crash 2024-09-17T02:00:50.856 DEBUG:teuthology.orchestra.run.smithi081:> sudo tar c -f - -C /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash -- . 2024-09-17T02:00:50.911 INFO:teuthology.orchestra.run.smithi081.stderr:tar: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T02:00:50.911 INFO:teuthology.orchestra.run.smithi081.stderr:tar: Error is not recoverable: exiting now 2024-09-17T02:00:50.913 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi105/crash 2024-09-17T02:00:50.913 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash -- . 2024-09-17T02:00:50.955 INFO:teuthology.orchestra.run.smithi105.stderr:tar: /var/lib/ceph/ded6f264-7492-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T02:00:50.956 INFO:teuthology.orchestra.run.smithi105.stderr:tar: Error is not recoverable: exiting now 2024-09-17T02:00:50.957 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T02:00:50.957 DEBUG:teuthology.orchestra.run.smithi081:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-09-17T02:00:50.990 INFO:tasks.cephadm:Compressing logs... 2024-09-17T02:00:50.991 DEBUG:teuthology.orchestra.run.smithi081:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T02:00:51.034 DEBUG:teuthology.orchestra.run.smithi105:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:: No such file or directory 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/tcmu-runner.log 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr: 87.8% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/tcmu-runner.log.gz 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T02:00:51.067 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi081.fnhdrk.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi081.fnhdrk.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr: 76.6% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi081.fnhdrk.log.gz 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.2.log: 93.9% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T02:00:51.068 INFO:teuthology.orchestra.run.smithi105.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi105.ttsowb.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:: No such file or directory 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi105.ttsowb.log: /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr: 76.1% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-client.rgw.foo.smithi105.ttsowb.log.gz 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T02:00:51.069 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.6.log: /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.5.log: 89.7% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T02:00:51.070 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log 2024-09-17T02:00:51.070 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-17T02:00:51.070 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-17T02:00:51.070 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.x.log: 91.1% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T02:00:51.070 INFO:teuthology.orchestra.run.smithi105.stderr: 89.2% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T02:00:51.071 INFO:teuthology.orchestra.run.smithi105.stderr: 90.9% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T02:00:51.072 INFO:teuthology.orchestra.run.smithi081.stderr: 94.4% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T02:00:51.073 INFO:teuthology.orchestra.run.smithi081.stderr:gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log 2024-09-17T02:00:51.073 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T02:00:51.074 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log: 90.8% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T02:00:51.074 INFO:teuthology.orchestra.run.smithi081.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T02:00:51.076 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T02:00:51.080 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.3.log: 94.0% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T02:00:51.097 INFO:teuthology.orchestra.run.smithi081.stderr:/var/log/ceph/cephadm.log: 91.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T02:00:51.108 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 90.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T02:00:51.215 INFO:teuthology.orchestra.run.smithi105.stderr: 90.2% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-17T02:00:51.542 INFO:teuthology.orchestra.run.smithi081.stderr: 89.6% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-17T02:00:51.584 INFO:teuthology.orchestra.run.smithi105.stderr: 92.5% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-17T02:00:51.748 INFO:teuthology.orchestra.run.smithi081.stderr: 92.5% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-17T02:00:52.578 INFO:teuthology.orchestra.run.smithi081.stderr: 91.0% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-17T02:00:52.710 INFO:teuthology.orchestra.run.smithi105.stderr: 93.2% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T02:00:52.716 INFO:teuthology.orchestra.run.smithi081.stderr: 93.3% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T02:00:52.717 INFO:teuthology.orchestra.run.smithi081.stderr: 93.5% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T02:00:52.828 INFO:teuthology.orchestra.run.smithi105.stderr: 93.4% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T02:00:53.021 INFO:teuthology.orchestra.run.smithi105.stderr: 93.8% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T02:00:53.077 INFO:teuthology.orchestra.run.smithi105.stderr: 93.6% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T02:00:53.078 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-17T02:00:53.079 INFO:teuthology.orchestra.run.smithi105.stderr:real 0m2.029s 2024-09-17T02:00:53.079 INFO:teuthology.orchestra.run.smithi105.stderr:user 0m7.814s 2024-09-17T02:00:53.079 INFO:teuthology.orchestra.run.smithi105.stderr:sys 0m0.308s 2024-09-17T02:00:53.377 INFO:teuthology.orchestra.run.smithi081.stderr: 93.5% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T02:00:53.427 INFO:teuthology.orchestra.run.smithi081.stderr: 93.5% -- replaced with /var/log/ceph/ded6f264-7492-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T02:00:53.428 INFO:teuthology.orchestra.run.smithi081.stderr: 2024-09-17T02:00:53.428 INFO:teuthology.orchestra.run.smithi081.stderr:real 0m2.380s 2024-09-17T02:00:53.429 INFO:teuthology.orchestra.run.smithi081.stderr:user 0m10.329s 2024-09-17T02:00:53.429 INFO:teuthology.orchestra.run.smithi081.stderr:sys 0m0.382s 2024-09-17T02:00:53.429 INFO:tasks.cephadm:Archiving logs... 2024-09-17T02:00:53.429 DEBUG:teuthology.misc:Transferring archived files from smithi081:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi081/log 2024-09-17T02:00:53.430 DEBUG:teuthology.orchestra.run.smithi081:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T02:00:54.810 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi105/log 2024-09-17T02:00:54.811 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T02:00:55.958 INFO:tasks.cephadm:Removing cluster... 2024-09-17T02:00:55.958 DEBUG:teuthology.orchestra.run.smithi081:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ded6f264-7492-11ef-bceb-c7b262605968 --force 2024-09-17T02:00:56.244 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ded6f264-7492-11ef-bceb-c7b262605968 --force 2024-09-17T02:00:56.529 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T02:00:56.529 DEBUG:teuthology.orchestra.run.smithi081:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T02:00:56.545 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T02:00:56.560 INFO:tasks.cephadm:Teardown complete 2024-09-17T02:00:56.561 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T02:00:56.570 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T02:00:56.571 DEBUG:teuthology.orchestra.run.smithi081:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T02:00:56.587 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T02:00:56.599 INFO:teuthology.orchestra.run.smithi081.stderr:bash: line 1: ntpq: command not found 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:=============================================================================== 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:^* hv01.front.sepia.ceph.com 3 7 377 32 +2905us[+2877us] +/- 62ms 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 29 -2935us[-2935us] +/- 77ms 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 30 -2137us[-2137us] +/- 79ms 2024-09-17T02:00:56.603 INFO:teuthology.orchestra.run.smithi081.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T02:00:56.615 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv01.front.sepia.ceph.com 3 6 377 27 +3392us[+3363us] +/- 62ms 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 28 -2447us[-2476us] +/- 77ms 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 31 -1625us[-1653us] +/- 79ms 2024-09-17T02:00:56.619 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T02:00:56.619 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T02:00:56.628 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T02:00:56.629 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T02:00:56.657 DEBUG:teuthology.orchestra.run.smithi081:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T02:00:56.743 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T02:00:56.818 DEBUG:teuthology.orchestra.run.smithi081:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T02:00:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T02:00:56.848 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T02:00:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T02:00:56.876 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T02:00:56.884 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T02:00:56.905 INFO:teuthology.task.internal:Duration was 2720.152838 seconds 2024-09-17T02:00:56.906 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T02:00:56.939 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T02:00:56.939 DEBUG:teuthology.orchestra.run.smithi081:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T02:00:56.942 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T02:00:56.977 INFO:teuthology.orchestra.run.smithi081.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T02:00:56.980 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T02:00:57.476 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T02:00:57.477 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi081.front.sepia.ceph.com 2024-09-17T02:00:57.477 DEBUG:teuthology.orchestra.run.smithi081:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T02:00:57.537 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-09-17T02:00:57.537 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T02:00:57.563 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T02:00:57.563 DEBUG:teuthology.orchestra.run.smithi081:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T02:00:57.580 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T02:00:57.630 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T02:00:57.630 DEBUG:teuthology.orchestra.run.smithi081:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T02:00:57.647 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T02:00:58.216 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T02:00:58.227 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T02:00:58.227 DEBUG:teuthology.orchestra.run.smithi081:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T02:00:58.251 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T02:00:58.276 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T02:00:58.288 DEBUG:teuthology.orchestra.run.smithi081:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T02:00:58.296 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T02:00:58.316 INFO:teuthology.orchestra.run.smithi081.stdout:kernel.core_pattern = core 2024-09-17T02:00:58.340 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-09-17T02:00:58.380 DEBUG:teuthology.orchestra.run.smithi081:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T02:00:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T02:00:58.401 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T02:00:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T02:00:58.435 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T02:00:58.445 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T02:00:58.446 DEBUG:teuthology.misc:Transferring archived files from smithi081:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi081 2024-09-17T02:00:58.446 DEBUG:teuthology.orchestra.run.smithi081:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T02:00:58.496 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7908031/remote/smithi105 2024-09-17T02:00:58.496 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T02:00:58.536 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T02:00:58.537 DEBUG:teuthology.orchestra.run.smithi081:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T02:00:58.539 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T02:00:58.584 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T02:00:58.595 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T02:00:58.595 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T02:00:58.615 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T02:00:58.615 DEBUG:teuthology.orchestra.run.smithi081:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T02:00:58.617 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T02:00:58.630 INFO:teuthology.orchestra.run.smithi081.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 02:00 /home/ubuntu/cephtest 2024-09-17T02:00:58.639 INFO:teuthology.orchestra.run.smithi105.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 02:00 /home/ubuntu/cephtest 2024-09-17T02:00:58.640 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T02:00:58.656 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T02:00:58.724 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2720.1528379917145 owner: scheduled_teuthology@teuthology success: true 2024-09-17T02:00:58.725 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T02:00:58.805 INFO:teuthology.run:pass